diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index fa3e671ba1..c5f961b8e2 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -4631,6 +4631,15 @@ L_curve25519_x64_bits: adc r11, r14 adc r12, r15 adc r13, rdi + mov rbp, 9223372036854775807 + mov rax, r13 + sar rax, 63 + and rax, 19 + and r13, rbp + add rcx, rax + adc r11, 0 + adc r12, 0 + adc r13, 0 ; Store mov QWORD PTR [rsp+64], rcx mov QWORD PTR [rsp+72], r11 @@ -14299,6 +14308,15 @@ L_curve25519_avx2_bits: adcx r13, rbx adox r14, rsi adcx r14, rcx + mov rcx, 9223372036854775807 + mov rdx, r14 + sar rdx, 63 + and rdx, 19 + and r14, rcx + add r11, rdx + adc r12, 0 + adc r13, 0 + adc r14, 0 ; Store mov QWORD PTR [rsp+64], r11 mov QWORD PTR [rsp+72], r12 diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.asm b/wolfcrypt/src/port/arm/armv8-aes-asm.asm index d202cded52..b6155ace79 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.asm @@ -41,10 +41,10 @@ AES_set_key_AARCH64 PROC lsr x5, x4, #32 lsr x7, x6, #32 lsr x9, x8, #32 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 @@ -56,10 +56,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 @@ -71,10 +71,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 @@ -86,10 +86,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 @@ -101,10 +101,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 @@ -116,10 +116,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 @@ -131,10 +131,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 @@ -146,10 +146,10 @@ AES_set_key_AARCH64 PROC stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 - dup v1.4s, w9 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w9 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #0x80 eor w4, w4, w12 @@ -161,43 +161,43 @@ AES_set_key_AARCH64 PROC cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xd0 - ldur q0, [x2] - ldur q1, [x2, #192] - stur q1, [x2] - stur q0, [x2, #192] - ldur q0, [x2, #16] - ldur q1, [x2, #176] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #16] - stur q0, [x2, #176] - ldur q0, [x2, #32] - ldur q1, [x2, #160] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #32] - stur q0, [x2, #160] - ldur q0, [x2, #48] - ldur q1, [x2, #144] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #48] - stur q0, [x2, #144] - ldur q0, [x2, #64] - ldur q1, [x2, #128] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #64] - stur q0, [x2, #128] - ldur q0, [x2, #80] - ldur q1, [x2, #112] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #80] - stur q0, [x2, #112] - ldur q0, [x2, #96] - aesimc v0.16b, v0.16b - stur q0, [x2, #96] + ldur Q0, [x2] + ldur Q1, [x2, #192] + stur Q1, [x2] + stur Q0, [x2, #192] + ldur Q0, [x2, #16] + ldur Q1, [x2, #176] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #16] + stur Q0, [x2, #176] + ldur Q0, [x2, #32] + ldur Q1, [x2, #160] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #32] + stur Q0, [x2, #160] + ldur Q0, [x2, #48] + ldur Q1, [x2, #144] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #48] + stur Q0, [x2, #144] + ldur Q0, [x2, #64] + ldur Q1, [x2, #128] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #64] + stur Q0, [x2, #128] + ldur Q0, [x2, #80] + ldur Q1, [x2, #112] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #80] + stur Q0, [x2, #112] + ldur Q0, [x2, #96] + aesimc V0.16B, V0.16B + stur Q0, [x2, #96] b L_aes_set_key_arm64_crypto_done L_aes_set_key_arm64_crypto_start_256 ldr x4, [x0], #8 @@ -210,20 +210,20 @@ L_aes_set_key_arm64_crypto_start_256 lsr x7, x6, #32 lsr x9, x8, #32 lsr x11, x10, #32 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -232,20 +232,20 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -254,20 +254,20 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -276,20 +276,20 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -298,20 +298,20 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -320,20 +320,20 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 eor w5, w5, w4 eor w6, w6, w5 eor w7, w7, w6 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] eor w8, w8, w12 eor w9, w9, w8 eor w10, w10, w9 @@ -342,10 +342,10 @@ L_aes_set_key_arm64_crypto_start_256 stp w6, w7, [x2], #8 stp w8, w9, [x2], #8 stp w10, w11, [x2], #8 - dup v1.4s, w11 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w11 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 @@ -357,49 +357,49 @@ L_aes_set_key_arm64_crypto_start_256 cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xf0 - ldur q0, [x2] - ldur q1, [x2, #224] - stur q1, [x2] - stur q0, [x2, #224] - ldur q0, [x2, #16] - ldur q1, [x2, #208] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #16] - stur q0, [x2, #208] - ldur q0, [x2, #32] - ldur q1, [x2, #192] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #32] - stur q0, [x2, #192] - ldur q0, [x2, #48] - ldur q1, [x2, #176] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #48] - stur q0, [x2, #176] - ldur q0, [x2, #64] - ldur q1, [x2, #160] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #64] - stur q0, [x2, #160] - ldur q0, [x2, #80] - ldur q1, [x2, #144] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #80] - stur q0, [x2, #144] - ldur q0, [x2, #96] - ldur q1, [x2, #128] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #96] - stur q0, [x2, #128] - ldur q0, [x2, #112] - aesimc v0.16b, v0.16b - stur q0, [x2, #112] + ldur Q0, [x2] + ldur Q1, [x2, #224] + stur Q1, [x2] + stur Q0, [x2, #224] + ldur Q0, [x2, #16] + ldur Q1, [x2, #208] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #16] + stur Q0, [x2, #208] + ldur Q0, [x2, #32] + ldur Q1, [x2, #192] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #32] + stur Q0, [x2, #192] + ldur Q0, [x2, #48] + ldur Q1, [x2, #176] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #48] + stur Q0, [x2, #176] + ldur Q0, [x2, #64] + ldur Q1, [x2, #160] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #64] + stur Q0, [x2, #160] + ldur Q0, [x2, #80] + ldur Q1, [x2, #144] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #80] + stur Q0, [x2, #144] + ldur Q0, [x2, #96] + ldur Q1, [x2, #128] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #96] + stur Q0, [x2, #128] + ldur Q0, [x2, #112] + aesimc V0.16B, V0.16B + stur Q0, [x2, #112] b L_aes_set_key_arm64_crypto_done L_aes_set_key_arm64_crypto_start_128 ldr x4, [x0], #8 @@ -407,10 +407,10 @@ L_aes_set_key_arm64_crypto_start_128 stp x4, x6, [x2], #16 lsr x5, x4, #32 lsr x7, x6, #32 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #1 eor w4, w4, w12 @@ -419,10 +419,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #2 eor w4, w4, w12 @@ -431,10 +431,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #4 eor w4, w4, w12 @@ -443,10 +443,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #8 eor w4, w4, w12 @@ -455,10 +455,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #16 eor w4, w4, w12 @@ -467,10 +467,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #32 eor w4, w4, w12 @@ -479,10 +479,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #0x40 eor w4, w4, w12 @@ -491,10 +491,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] ror w12, w12, #8 eor w4, w4, #0x80 eor w4, w4, w12 @@ -503,10 +503,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] mov w13, #27 ror w12, w12, #8 eor w4, w4, w13 @@ -516,10 +516,10 @@ L_aes_set_key_arm64_crypto_start_128 eor w7, w7, w6 stp w4, w5, [x2], #8 stp w6, w7, [x2], #8 - dup v1.4s, w7 - movi v0.16b, #0 - aese v0.16b, v1.16b - mov w12, v0.s[0] + dup V1.4S, w7 + movi V0.16B, #0 + aese V0.16B, V1.16B + mov w12, V0.S[0] mov w13, #54 ror w12, w12, #8 eor w4, w4, w13 @@ -532,37 +532,37 @@ L_aes_set_key_arm64_crypto_start_128 cmp x3, #0 beq L_aes_set_key_arm64_crypto_done sub x2, x2, #0xb0 - ldur q0, [x2] - ldur q1, [x2, #160] - stur q1, [x2] - stur q0, [x2, #160] - ldur q0, [x2, #16] - ldur q1, [x2, #144] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #16] - stur q0, [x2, #144] - ldur q0, [x2, #32] - ldur q1, [x2, #128] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #32] - stur q0, [x2, #128] - ldur q0, [x2, #48] - ldur q1, [x2, #112] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #48] - stur q0, [x2, #112] - ldur q0, [x2, #64] - ldur q1, [x2, #96] - aesimc v0.16b, v0.16b - aesimc v1.16b, v1.16b - stur q1, [x2, #64] - stur q0, [x2, #96] - ldur q0, [x2, #80] - aesimc v0.16b, v0.16b - stur q0, [x2, #80] + ldur Q0, [x2] + ldur Q1, [x2, #160] + stur Q1, [x2] + stur Q0, [x2, #160] + ldur Q0, [x2, #16] + ldur Q1, [x2, #144] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #16] + stur Q0, [x2, #144] + ldur Q0, [x2, #32] + ldur Q1, [x2, #128] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #32] + stur Q0, [x2, #128] + ldur Q0, [x2, #48] + ldur Q1, [x2, #112] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #48] + stur Q0, [x2, #112] + ldur Q0, [x2, #64] + ldur Q1, [x2, #96] + aesimc V0.16B, V0.16B + aesimc V1.16B, V1.16B + stur Q1, [x2, #64] + stur Q0, [x2, #96] + ldur Q0, [x2, #80] + aesimc V0.16B, V0.16B + stur Q0, [x2, #80] L_aes_set_key_arm64_crypto_done ret ENDP @@ -571,47 +571,47 @@ L_aes_set_key_arm64_crypto_done ALIGN 4 EXPORT AES_encrypt_AARCH64 AES_encrypt_AARCH64 PROC - ld1 {v0.16b}, [x0] - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b - aesmc v0.16b, v0.16b - aese v0.16b, v3.16b - aesmc v0.16b, v0.16b - aese v0.16b, v4.16b - aesmc v0.16b, v0.16b - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b - aesmc v0.16b, v0.16b - aese v0.16b, v3.16b - aesmc v0.16b, v0.16b - aese v0.16b, v4.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B}, [x0] + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x2], #0x40 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B + aesmc V0.16B, V0.16B + aese V0.16B, V3.16B + aesmc V0.16B, V0.16B + aese V0.16B, V4.16B + aesmc V0.16B, V0.16B + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x2], #0x40 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B + aesmc V0.16B, V0.16B + aese V0.16B, V3.16B + aesmc V0.16B, V0.16B + aese V0.16B, V4.16B + aesmc V0.16B, V0.16B subs w3, w3, #10 - ld1 {v1.2d, v2.2d}, [x2], #32 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x2], #32 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B beq L_aes_encrypt_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x2], #32 + ld1 {V1.2D, V2.2D}, [x2], #32 subs w3, w3, #2 - aesmc v0.16b, v0.16b - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + aesmc V0.16B, V0.16B + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B beq L_aes_encrypt_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x2], #32 - aesmc v0.16b, v0.16b - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x2], #32 + aesmc V0.16B, V0.16B + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B L_aes_encrypt_arm64_crypto_round_done - ld1 {v1.2d}, [x2] - eor v0.16b, v0.16b, v1.16b - st1 {v0.16b}, [x1] + ld1 {V1.2D}, [x2] + eor V0.16B, V0.16B, V1.16B + st1 {V0.16B}, [x1] ret ENDP ENDIF @@ -621,47 +621,47 @@ L_aes_encrypt_arm64_crypto_round_done ALIGN 4 EXPORT AES_decrypt_AARCH64 AES_decrypt_AARCH64 PROC - ld1 {v0.16b}, [x0] - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 - aesd v0.16b, v1.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v2.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v3.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v4.16b - aesimc v0.16b, v0.16b - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x2], #0x40 - aesd v0.16b, v1.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v2.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v3.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v4.16b - aesimc v0.16b, v0.16b - ld1 {v1.2d, v2.2d}, [x2], #32 - aesd v0.16b, v1.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v2.16b + ld1 {V0.16B}, [x0] + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x2], #0x40 + aesd V0.16B, V1.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V2.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V3.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V4.16B + aesimc V0.16B, V0.16B + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x2], #0x40 + aesd V0.16B, V1.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V2.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V3.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V4.16B + aesimc V0.16B, V0.16B + ld1 {V1.2D, V2.2D}, [x2], #32 + aesd V0.16B, V1.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V2.16B subs w3, w3, #10 beq L_aes_decrypt_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x2], #32 - aesimc v0.16b, v0.16b - aesd v0.16b, v1.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x2], #32 + aesimc V0.16B, V0.16B + aesd V0.16B, V1.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V2.16B subs w3, w3, #2 beq L_aes_decrypt_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x2], #32 - aesimc v0.16b, v0.16b - aesd v0.16b, v1.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x2], #32 + aesimc V0.16B, V0.16B + aesd V0.16B, V1.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V2.16B L_aes_decrypt_arm64_crypto_round_done - ld1 {v1.2d}, [x2] - eor v0.16b, v0.16b, v1.16b - st1 {v0.16b}, [x1] + ld1 {V1.2D}, [x2] + eor V0.16B, V0.16B, V1.16B + st1 {V0.16B}, [x1] ret ENDP ENDIF @@ -673,859 +673,859 @@ L_aes_decrypt_arm64_crypto_round_done AES_encrypt_blocks_AARCH64 PROC stp x29, x30, [sp, #-32]! add x29, sp, #0 - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 - ld1 {v24.2d, v25.2d, v26.2d}, [x3], #48 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x3], #0x40 + ld1 {V24.2D, V25.2D, V26.2D}, [x3], #48 lsr w2, w2, #4 cmp w4, #12 blt L_aes_encrypt_blocks_arm64_crypto_start_128 bgt L_aes_encrypt_blocks_arm64_crypto_start_256 ; AES_ECB_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v27.2d, v28.2d}, [x3], #32 + ld1 {V27.2D, V28.2D}, [x3], #32 cmp w2, #1 beq L_aes_encrypt_blocks_arm64_crypto_192_start_1 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_192_start_4 L_aes_encrypt_blocks_arm64_crypto_192_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v5.16b, v16.16b - aesmc v5.16b, v5.16b - aese v6.16b, v16.16b - aesmc v6.16b, v6.16b - aese v7.16b, v16.16b - aesmc v7.16b, v7.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v5.16b, v17.16b - aesmc v5.16b, v5.16b - aese v6.16b, v17.16b - aesmc v6.16b, v6.16b - aese v7.16b, v17.16b - aesmc v7.16b, v7.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v5.16b, v18.16b - aesmc v5.16b, v5.16b - aese v6.16b, v18.16b - aesmc v6.16b, v6.16b - aese v7.16b, v18.16b - aesmc v7.16b, v7.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v5.16b, v19.16b - aesmc v5.16b, v5.16b - aese v6.16b, v19.16b - aesmc v6.16b, v6.16b - aese v7.16b, v19.16b - aesmc v7.16b, v7.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v5.16b, v20.16b - aesmc v5.16b, v5.16b - aese v6.16b, v20.16b - aesmc v6.16b, v6.16b - aese v7.16b, v20.16b - aesmc v7.16b, v7.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v5.16b, v21.16b - aesmc v5.16b, v5.16b - aese v6.16b, v21.16b - aesmc v6.16b, v6.16b - aese v7.16b, v21.16b - aesmc v7.16b, v7.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v5.16b, v22.16b - aesmc v5.16b, v5.16b - aese v6.16b, v22.16b - aesmc v6.16b, v6.16b - aese v7.16b, v22.16b - aesmc v7.16b, v7.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v5.16b, v23.16b - aesmc v5.16b, v5.16b - aese v6.16b, v23.16b - aesmc v6.16b, v6.16b - aese v7.16b, v23.16b - aesmc v7.16b, v7.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v5.16b, v24.16b - aesmc v5.16b, v5.16b - aese v6.16b, v24.16b - aesmc v6.16b, v6.16b - aese v7.16b, v24.16b - aesmc v7.16b, v7.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v5.16b, v25.16b - aesmc v5.16b, v5.16b - aese v6.16b, v25.16b - aesmc v6.16b, v6.16b - aese v7.16b, v25.16b - aesmc v7.16b, v7.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aese v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aese v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aese v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b - aese v4.16b, v27.16b - eor v4.16b, v4.16b, v28.16b - aese v5.16b, v27.16b - eor v5.16b, v5.16b, v28.16b - aese v6.16b, v27.16b - eor v6.16b, v6.16b, v28.16b - aese v7.16b, v27.16b - eor v7.16b, v7.16b, v28.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V5.16B, V16.16B + aesmc V5.16B, V5.16B + aese V6.16B, V16.16B + aesmc V6.16B, V6.16B + aese V7.16B, V16.16B + aesmc V7.16B, V7.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V5.16B, V17.16B + aesmc V5.16B, V5.16B + aese V6.16B, V17.16B + aesmc V6.16B, V6.16B + aese V7.16B, V17.16B + aesmc V7.16B, V7.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V5.16B, V18.16B + aesmc V5.16B, V5.16B + aese V6.16B, V18.16B + aesmc V6.16B, V6.16B + aese V7.16B, V18.16B + aesmc V7.16B, V7.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V5.16B, V19.16B + aesmc V5.16B, V5.16B + aese V6.16B, V19.16B + aesmc V6.16B, V6.16B + aese V7.16B, V19.16B + aesmc V7.16B, V7.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V5.16B, V20.16B + aesmc V5.16B, V5.16B + aese V6.16B, V20.16B + aesmc V6.16B, V6.16B + aese V7.16B, V20.16B + aesmc V7.16B, V7.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V5.16B, V21.16B + aesmc V5.16B, V5.16B + aese V6.16B, V21.16B + aesmc V6.16B, V6.16B + aese V7.16B, V21.16B + aesmc V7.16B, V7.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V5.16B, V22.16B + aesmc V5.16B, V5.16B + aese V6.16B, V22.16B + aesmc V6.16B, V6.16B + aese V7.16B, V22.16B + aesmc V7.16B, V7.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V5.16B, V23.16B + aesmc V5.16B, V5.16B + aese V6.16B, V23.16B + aesmc V6.16B, V6.16B + aese V7.16B, V23.16B + aesmc V7.16B, V7.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V5.16B, V24.16B + aesmc V5.16B, V5.16B + aese V6.16B, V24.16B + aesmc V6.16B, V6.16B + aese V7.16B, V24.16B + aesmc V7.16B, V7.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V5.16B, V25.16B + aesmc V5.16B, V5.16B + aese V6.16B, V25.16B + aesmc V6.16B, V6.16B + aese V7.16B, V25.16B + aesmc V7.16B, V7.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V5.16B, V26.16B + aesmc V5.16B, V5.16B + aese V6.16B, V26.16B + aesmc V6.16B, V6.16B + aese V7.16B, V26.16B + aesmc V7.16B, V7.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aese V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aese V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aese V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B + aese V4.16B, V27.16B + eor V4.16B, V4.16B, V28.16B + aese V5.16B, V27.16B + eor V5.16B, V5.16B, V28.16B + aese V6.16B, V27.16B + eor V6.16B, V6.16B, V28.16B + aese V7.16B, V27.16B + eor V7.16B, V7.16B, V28.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_192_start_8 L_aes_encrypt_blocks_arm64_crypto_192_start_4 cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_192_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aese v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aese v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aese v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aese V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aese V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aese V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_192_start_2 cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_192_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aese v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aese V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_192_start_1 cbz w2, L_aes_encrypt_blocks_arm64_crypto_192_done - ld1 {v0.16b}, [x0], #16 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + st1 {V0.16B}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_192_done ENDIF b L_aes_encrypt_blocks_arm64_crypto_done ; AES_ECB_256 L_aes_encrypt_blocks_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v27.2d, v28.2d, v29.2d, v30.2d}, [x3], #0x40 + ld1 {V27.2D, V28.2D, V29.2D, V30.2D}, [x3], #0x40 cmp w2, #1 beq L_aes_encrypt_blocks_arm64_crypto_256_start_1 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_256_start_4 L_aes_encrypt_blocks_arm64_crypto_256_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v5.16b, v16.16b - aesmc v5.16b, v5.16b - aese v6.16b, v16.16b - aesmc v6.16b, v6.16b - aese v7.16b, v16.16b - aesmc v7.16b, v7.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v5.16b, v17.16b - aesmc v5.16b, v5.16b - aese v6.16b, v17.16b - aesmc v6.16b, v6.16b - aese v7.16b, v17.16b - aesmc v7.16b, v7.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v5.16b, v18.16b - aesmc v5.16b, v5.16b - aese v6.16b, v18.16b - aesmc v6.16b, v6.16b - aese v7.16b, v18.16b - aesmc v7.16b, v7.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v5.16b, v19.16b - aesmc v5.16b, v5.16b - aese v6.16b, v19.16b - aesmc v6.16b, v6.16b - aese v7.16b, v19.16b - aesmc v7.16b, v7.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v5.16b, v20.16b - aesmc v5.16b, v5.16b - aese v6.16b, v20.16b - aesmc v6.16b, v6.16b - aese v7.16b, v20.16b - aesmc v7.16b, v7.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v5.16b, v21.16b - aesmc v5.16b, v5.16b - aese v6.16b, v21.16b - aesmc v6.16b, v6.16b - aese v7.16b, v21.16b - aesmc v7.16b, v7.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v5.16b, v22.16b - aesmc v5.16b, v5.16b - aese v6.16b, v22.16b - aesmc v6.16b, v6.16b - aese v7.16b, v22.16b - aesmc v7.16b, v7.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v5.16b, v23.16b - aesmc v5.16b, v5.16b - aese v6.16b, v23.16b - aesmc v6.16b, v6.16b - aese v7.16b, v23.16b - aesmc v7.16b, v7.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v5.16b, v24.16b - aesmc v5.16b, v5.16b - aese v6.16b, v24.16b - aesmc v6.16b, v6.16b - aese v7.16b, v24.16b - aesmc v7.16b, v7.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v5.16b, v25.16b - aesmc v5.16b, v5.16b - aese v6.16b, v25.16b - aesmc v6.16b, v6.16b - aese v7.16b, v25.16b - aesmc v7.16b, v7.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v5.16b, v26.16b - aesmc v5.16b, v5.16b - aese v6.16b, v26.16b - aesmc v6.16b, v6.16b - aese v7.16b, v26.16b - aesmc v7.16b, v7.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b - aese v5.16b, v27.16b - aesmc v5.16b, v5.16b - aese v6.16b, v27.16b - aesmc v6.16b, v6.16b - aese v7.16b, v27.16b - aesmc v7.16b, v7.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b - aese v5.16b, v28.16b - aesmc v5.16b, v5.16b - aese v6.16b, v28.16b - aesmc v6.16b, v6.16b - aese v7.16b, v28.16b - aesmc v7.16b, v7.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aese v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aese v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aese v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b - aese v4.16b, v29.16b - eor v4.16b, v4.16b, v30.16b - aese v5.16b, v29.16b - eor v5.16b, v5.16b, v30.16b - aese v6.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - aese v7.16b, v29.16b - eor v7.16b, v7.16b, v30.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V5.16B, V16.16B + aesmc V5.16B, V5.16B + aese V6.16B, V16.16B + aesmc V6.16B, V6.16B + aese V7.16B, V16.16B + aesmc V7.16B, V7.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V5.16B, V17.16B + aesmc V5.16B, V5.16B + aese V6.16B, V17.16B + aesmc V6.16B, V6.16B + aese V7.16B, V17.16B + aesmc V7.16B, V7.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V5.16B, V18.16B + aesmc V5.16B, V5.16B + aese V6.16B, V18.16B + aesmc V6.16B, V6.16B + aese V7.16B, V18.16B + aesmc V7.16B, V7.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V5.16B, V19.16B + aesmc V5.16B, V5.16B + aese V6.16B, V19.16B + aesmc V6.16B, V6.16B + aese V7.16B, V19.16B + aesmc V7.16B, V7.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V5.16B, V20.16B + aesmc V5.16B, V5.16B + aese V6.16B, V20.16B + aesmc V6.16B, V6.16B + aese V7.16B, V20.16B + aesmc V7.16B, V7.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V5.16B, V21.16B + aesmc V5.16B, V5.16B + aese V6.16B, V21.16B + aesmc V6.16B, V6.16B + aese V7.16B, V21.16B + aesmc V7.16B, V7.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V5.16B, V22.16B + aesmc V5.16B, V5.16B + aese V6.16B, V22.16B + aesmc V6.16B, V6.16B + aese V7.16B, V22.16B + aesmc V7.16B, V7.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V5.16B, V23.16B + aesmc V5.16B, V5.16B + aese V6.16B, V23.16B + aesmc V6.16B, V6.16B + aese V7.16B, V23.16B + aesmc V7.16B, V7.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V5.16B, V24.16B + aesmc V5.16B, V5.16B + aese V6.16B, V24.16B + aesmc V6.16B, V6.16B + aese V7.16B, V24.16B + aesmc V7.16B, V7.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V5.16B, V25.16B + aesmc V5.16B, V5.16B + aese V6.16B, V25.16B + aesmc V6.16B, V6.16B + aese V7.16B, V25.16B + aesmc V7.16B, V7.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V5.16B, V26.16B + aesmc V5.16B, V5.16B + aese V6.16B, V26.16B + aesmc V6.16B, V6.16B + aese V7.16B, V26.16B + aesmc V7.16B, V7.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V1.16B, V27.16B + aesmc V1.16B, V1.16B + aese V2.16B, V27.16B + aesmc V2.16B, V2.16B + aese V3.16B, V27.16B + aesmc V3.16B, V3.16B + aese V4.16B, V27.16B + aesmc V4.16B, V4.16B + aese V5.16B, V27.16B + aesmc V5.16B, V5.16B + aese V6.16B, V27.16B + aesmc V6.16B, V6.16B + aese V7.16B, V27.16B + aesmc V7.16B, V7.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V1.16B, V28.16B + aesmc V1.16B, V1.16B + aese V2.16B, V28.16B + aesmc V2.16B, V2.16B + aese V3.16B, V28.16B + aesmc V3.16B, V3.16B + aese V4.16B, V28.16B + aesmc V4.16B, V4.16B + aese V5.16B, V28.16B + aesmc V5.16B, V5.16B + aese V6.16B, V28.16B + aesmc V6.16B, V6.16B + aese V7.16B, V28.16B + aesmc V7.16B, V7.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aese V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aese V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aese V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B + aese V4.16B, V29.16B + eor V4.16B, V4.16B, V30.16B + aese V5.16B, V29.16B + eor V5.16B, V5.16B, V30.16B + aese V6.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + aese V7.16B, V29.16B + eor V7.16B, V7.16B, V30.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_256_start_8 L_aes_encrypt_blocks_arm64_crypto_256_start_4 cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_256_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aese v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aese v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aese v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V1.16B, V27.16B + aesmc V1.16B, V1.16B + aese V2.16B, V27.16B + aesmc V2.16B, V2.16B + aese V3.16B, V27.16B + aesmc V3.16B, V3.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V1.16B, V28.16B + aesmc V1.16B, V1.16B + aese V2.16B, V28.16B + aesmc V2.16B, V2.16B + aese V3.16B, V28.16B + aesmc V3.16B, V3.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aese V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aese V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aese V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_256_start_2 cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_256_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aese v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V1.16B, V27.16B + aesmc V1.16B, V1.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V1.16B, V28.16B + aesmc V1.16B, V1.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aese V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_256_start_1 cbz w2, L_aes_encrypt_blocks_arm64_crypto_256_done - ld1 {v0.16b}, [x0], #16 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + st1 {V0.16B}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_256_done ENDIF b L_aes_encrypt_blocks_arm64_crypto_done @@ -1537,329 +1537,329 @@ L_aes_encrypt_blocks_arm64_crypto_start_128 cmp w2, #8 blt L_aes_encrypt_blocks_arm64_crypto_128_start_4 L_aes_encrypt_blocks_arm64_crypto_128_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v5.16b, v16.16b - aesmc v5.16b, v5.16b - aese v6.16b, v16.16b - aesmc v6.16b, v6.16b - aese v7.16b, v16.16b - aesmc v7.16b, v7.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v5.16b, v17.16b - aesmc v5.16b, v5.16b - aese v6.16b, v17.16b - aesmc v6.16b, v6.16b - aese v7.16b, v17.16b - aesmc v7.16b, v7.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v5.16b, v18.16b - aesmc v5.16b, v5.16b - aese v6.16b, v18.16b - aesmc v6.16b, v6.16b - aese v7.16b, v18.16b - aesmc v7.16b, v7.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v5.16b, v19.16b - aesmc v5.16b, v5.16b - aese v6.16b, v19.16b - aesmc v6.16b, v6.16b - aese v7.16b, v19.16b - aesmc v7.16b, v7.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v5.16b, v20.16b - aesmc v5.16b, v5.16b - aese v6.16b, v20.16b - aesmc v6.16b, v6.16b - aese v7.16b, v20.16b - aesmc v7.16b, v7.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v5.16b, v21.16b - aesmc v5.16b, v5.16b - aese v6.16b, v21.16b - aesmc v6.16b, v6.16b - aese v7.16b, v21.16b - aesmc v7.16b, v7.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v5.16b, v22.16b - aesmc v5.16b, v5.16b - aese v6.16b, v22.16b - aesmc v6.16b, v6.16b - aese v7.16b, v22.16b - aesmc v7.16b, v7.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v5.16b, v23.16b - aesmc v5.16b, v5.16b - aese v6.16b, v23.16b - aesmc v6.16b, v6.16b - aese v7.16b, v23.16b - aesmc v7.16b, v7.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v5.16b, v24.16b - aesmc v5.16b, v5.16b - aese v6.16b, v24.16b - aesmc v6.16b, v6.16b - aese v7.16b, v24.16b - aesmc v7.16b, v7.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aese v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aese v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aese v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b - aese v4.16b, v25.16b - eor v4.16b, v4.16b, v26.16b - aese v5.16b, v25.16b - eor v5.16b, v5.16b, v26.16b - aese v6.16b, v25.16b - eor v6.16b, v6.16b, v26.16b - aese v7.16b, v25.16b - eor v7.16b, v7.16b, v26.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V5.16B, V16.16B + aesmc V5.16B, V5.16B + aese V6.16B, V16.16B + aesmc V6.16B, V6.16B + aese V7.16B, V16.16B + aesmc V7.16B, V7.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V5.16B, V17.16B + aesmc V5.16B, V5.16B + aese V6.16B, V17.16B + aesmc V6.16B, V6.16B + aese V7.16B, V17.16B + aesmc V7.16B, V7.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V5.16B, V18.16B + aesmc V5.16B, V5.16B + aese V6.16B, V18.16B + aesmc V6.16B, V6.16B + aese V7.16B, V18.16B + aesmc V7.16B, V7.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V5.16B, V19.16B + aesmc V5.16B, V5.16B + aese V6.16B, V19.16B + aesmc V6.16B, V6.16B + aese V7.16B, V19.16B + aesmc V7.16B, V7.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V5.16B, V20.16B + aesmc V5.16B, V5.16B + aese V6.16B, V20.16B + aesmc V6.16B, V6.16B + aese V7.16B, V20.16B + aesmc V7.16B, V7.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V5.16B, V21.16B + aesmc V5.16B, V5.16B + aese V6.16B, V21.16B + aesmc V6.16B, V6.16B + aese V7.16B, V21.16B + aesmc V7.16B, V7.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V5.16B, V22.16B + aesmc V5.16B, V5.16B + aese V6.16B, V22.16B + aesmc V6.16B, V6.16B + aese V7.16B, V22.16B + aesmc V7.16B, V7.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V5.16B, V23.16B + aesmc V5.16B, V5.16B + aese V6.16B, V23.16B + aesmc V6.16B, V6.16B + aese V7.16B, V23.16B + aesmc V7.16B, V7.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V5.16B, V24.16B + aesmc V5.16B, V5.16B + aese V6.16B, V24.16B + aesmc V6.16B, V6.16B + aese V7.16B, V24.16B + aesmc V7.16B, V7.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aese V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aese V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aese V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B + aese V4.16B, V25.16B + eor V4.16B, V4.16B, V26.16B + aese V5.16B, V25.16B + eor V5.16B, V5.16B, V26.16B + aese V6.16B, V25.16B + eor V6.16B, V6.16B, V26.16B + aese V7.16B, V25.16B + eor V7.16B, V7.16B, V26.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_encrypt_blocks_arm64_crypto_128_start_8 L_aes_encrypt_blocks_arm64_crypto_128_start_4 cmp w2, #4 blt L_aes_encrypt_blocks_arm64_crypto_128_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aese v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aese v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aese v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aese V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aese V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aese V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_encrypt_blocks_arm64_crypto_128_start_2 cmp w2, #2 blt L_aes_encrypt_blocks_arm64_crypto_128_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aese v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aese V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_encrypt_blocks_arm64_crypto_128_start_1 cbz w2, L_aes_encrypt_blocks_arm64_crypto_128_done - ld1 {v0.16b}, [x0], #16 - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + st1 {V0.16B}, [x1], #16 L_aes_encrypt_blocks_arm64_crypto_128_done ENDIF L_aes_encrypt_blocks_arm64_crypto_done @@ -1873,859 +1873,859 @@ L_aes_encrypt_blocks_arm64_crypto_done AES_decrypt_blocks_AARCH64 PROC stp x29, x30, [sp, #-32]! add x29, sp, #0 - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x3], #0x40 - ld1 {v24.2d, v25.2d, v26.2d}, [x3], #48 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x3], #0x40 + ld1 {V24.2D, V25.2D, V26.2D}, [x3], #48 lsr w2, w2, #4 cmp w4, #12 blt L_aes_decrypt_blocks_arm64_crypto_start_128 bgt L_aes_decrypt_blocks_arm64_crypto_start_256 ; AES_ECB_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v27.2d, v28.2d}, [x3], #32 + ld1 {V27.2D, V28.2D}, [x3], #32 cmp w2, #1 beq L_aes_decrypt_blocks_arm64_crypto_192_start_1 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_192_start_4 L_aes_decrypt_blocks_arm64_crypto_192_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v16.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v16.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v16.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v16.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v17.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v17.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v17.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v17.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v18.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v18.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v18.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v18.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v19.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v19.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v19.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v19.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v20.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v20.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v20.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v20.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v21.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v21.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v21.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v21.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v22.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v22.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v22.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v22.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v23.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v23.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v23.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v23.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v24.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v24.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v24.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v24.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v25.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v25.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v25.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v25.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v26.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v26.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v26.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v26.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aesd v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aesd v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b - aesd v4.16b, v27.16b - eor v4.16b, v4.16b, v28.16b - aesd v5.16b, v27.16b - eor v5.16b, v5.16b, v28.16b - aesd v6.16b, v27.16b - eor v6.16b, v6.16b, v28.16b - aesd v7.16b, v27.16b - eor v7.16b, v7.16b, v28.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V16.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V16.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V16.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V16.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V17.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V17.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V17.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V17.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V18.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V18.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V18.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V18.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V19.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V19.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V19.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V19.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V20.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V20.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V20.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V20.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V21.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V21.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V21.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V21.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V22.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V22.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V22.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V22.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V23.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V23.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V23.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V23.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V24.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V24.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V24.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V24.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V25.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V25.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V25.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V25.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V26.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V26.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V26.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V26.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aesd V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aesd V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B + aesd V4.16B, V27.16B + eor V4.16B, V4.16B, V28.16B + aesd V5.16B, V27.16B + eor V5.16B, V5.16B, V28.16B + aesd V6.16B, V27.16B + eor V6.16B, V6.16B, V28.16B + aesd V7.16B, V27.16B + eor V7.16B, V7.16B, V28.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_192_start_8 L_aes_decrypt_blocks_arm64_crypto_192_start_4 cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_192_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aesd v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aesd v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aesd V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aesd V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_192_start_2 cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_192_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_192_start_1 cbz w2, L_aes_decrypt_blocks_arm64_crypto_192_done - ld1 {v0.16b}, [x0], #16 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + st1 {V0.16B}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_192_done ENDIF b L_aes_decrypt_blocks_arm64_crypto_done ; AES_ECB_256 L_aes_decrypt_blocks_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v27.2d, v28.2d, v29.2d, v30.2d}, [x3], #0x40 + ld1 {V27.2D, V28.2D, V29.2D, V30.2D}, [x3], #0x40 cmp w2, #1 beq L_aes_decrypt_blocks_arm64_crypto_256_start_1 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_256_start_4 L_aes_decrypt_blocks_arm64_crypto_256_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v16.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v16.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v16.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v16.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v17.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v17.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v17.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v17.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v18.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v18.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v18.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v18.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v19.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v19.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v19.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v19.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v20.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v20.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v20.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v20.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v21.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v21.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v21.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v21.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v22.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v22.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v22.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v22.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v23.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v23.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v23.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v23.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v24.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v24.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v24.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v24.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v25.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v25.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v25.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v25.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v26.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v26.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v26.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v26.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v27.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v27.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v27.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v27.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v27.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v27.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v28.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v28.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v28.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v28.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v28.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v28.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aesd v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aesd v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b - aesd v4.16b, v29.16b - eor v4.16b, v4.16b, v30.16b - aesd v5.16b, v29.16b - eor v5.16b, v5.16b, v30.16b - aesd v6.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - aesd v7.16b, v29.16b - eor v7.16b, v7.16b, v30.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V16.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V16.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V16.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V16.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V17.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V17.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V17.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V17.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V18.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V18.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V18.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V18.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V19.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V19.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V19.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V19.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V20.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V20.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V20.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V20.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V21.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V21.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V21.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V21.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V22.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V22.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V22.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V22.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V23.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V23.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V23.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V23.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V24.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V24.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V24.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V24.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V25.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V25.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V25.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V25.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V26.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V26.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V26.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V26.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V27.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V27.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V27.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V27.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V27.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V27.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V28.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V28.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V28.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V28.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V28.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V28.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aesd V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aesd V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B + aesd V4.16B, V29.16B + eor V4.16B, V4.16B, V30.16B + aesd V5.16B, V29.16B + eor V5.16B, V5.16B, V30.16B + aesd V6.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + aesd V7.16B, V29.16B + eor V7.16B, V7.16B, V30.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_256_start_8 L_aes_decrypt_blocks_arm64_crypto_256_start_4 cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_256_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v27.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v27.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v28.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v28.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aesd v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aesd v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V27.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V27.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V28.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V28.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aesd V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aesd V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_256_start_2 cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_256_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_256_start_1 cbz w2, L_aes_decrypt_blocks_arm64_crypto_256_done - ld1 {v0.16b}, [x0], #16 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + st1 {V0.16B}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_256_done ENDIF b L_aes_decrypt_blocks_arm64_crypto_done @@ -2737,329 +2737,329 @@ L_aes_decrypt_blocks_arm64_crypto_start_128 cmp w2, #8 blt L_aes_decrypt_blocks_arm64_crypto_128_start_4 L_aes_decrypt_blocks_arm64_crypto_128_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v16.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v16.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v16.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v16.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v17.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v17.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v17.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v17.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v18.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v18.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v18.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v18.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v19.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v19.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v19.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v19.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v20.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v20.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v20.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v20.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v21.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v21.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v21.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v21.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v22.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v22.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v22.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v22.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v23.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v23.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v23.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v23.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v4.16b, v24.16b - aesimc v4.16b, v4.16b - aesd v5.16b, v24.16b - aesimc v5.16b, v5.16b - aesd v6.16b, v24.16b - aesimc v6.16b, v6.16b - aesd v7.16b, v24.16b - aesimc v7.16b, v7.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aesd v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aesd v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b - aesd v4.16b, v25.16b - eor v4.16b, v4.16b, v26.16b - aesd v5.16b, v25.16b - eor v5.16b, v5.16b, v26.16b - aesd v6.16b, v25.16b - eor v6.16b, v6.16b, v26.16b - aesd v7.16b, v25.16b - eor v7.16b, v7.16b, v26.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V16.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V16.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V16.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V16.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V17.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V17.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V17.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V17.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V18.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V18.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V18.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V18.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V19.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V19.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V19.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V19.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V20.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V20.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V20.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V20.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V21.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V21.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V21.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V21.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V22.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V22.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V22.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V22.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V23.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V23.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V23.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V23.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V4.16B, V24.16B + aesimc V4.16B, V4.16B + aesd V5.16B, V24.16B + aesimc V5.16B, V5.16B + aesd V6.16B, V24.16B + aesimc V6.16B, V6.16B + aesd V7.16B, V24.16B + aesimc V7.16B, V7.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aesd V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aesd V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B + aesd V4.16B, V25.16B + eor V4.16B, V4.16B, V26.16B + aesd V5.16B, V25.16B + eor V5.16B, V5.16B, V26.16B + aesd V6.16B, V25.16B + eor V6.16B, V6.16B, V26.16B + aesd V7.16B, V25.16B + eor V7.16B, V7.16B, V26.16B sub w2, w2, #8 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 cmp w2, #8 bge L_aes_decrypt_blocks_arm64_crypto_128_start_8 L_aes_decrypt_blocks_arm64_crypto_128_start_4 cmp w2, #4 blt L_aes_decrypt_blocks_arm64_crypto_128_start_2 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aesd v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aesd v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aesd V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aesd V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B sub w2, w2, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 L_aes_decrypt_blocks_arm64_crypto_128_start_2 cmp w2, #2 blt L_aes_decrypt_blocks_arm64_crypto_128_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B sub w2, w2, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_decrypt_blocks_arm64_crypto_128_start_1 cbz w2, L_aes_decrypt_blocks_arm64_crypto_128_done - ld1 {v0.16b}, [x0], #16 - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - st1 {v0.16b}, [x1], #16 + ld1 {V0.16B}, [x0], #16 + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + st1 {V0.16B}, [x1], #16 L_aes_decrypt_blocks_arm64_crypto_128_done ENDIF L_aes_decrypt_blocks_arm64_crypto_done @@ -3073,125 +3073,125 @@ L_aes_decrypt_blocks_arm64_crypto_done ALIGN 4 EXPORT AES_CBC_encrypt_AARCH64 AES_CBC_encrypt_AARCH64 PROC - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v0.2d}, [x3] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V0.2D}, [x3] subs w5, w5, #12 lsr w2, w2, #4 blt L_aes_cbc_encrypt_arm64_crypto_start_128 bgt L_aes_cbc_encrypt_arm64_crypto_start_256 ; AES_CBC_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 L_aes_cbc_encrypt_arm64_crypto_loop_192 - ld1 {v28.2d}, [x4] - ld1 {v1.16b}, [x0], #16 + ld1 {V28.2D}, [x4] + ld1 {V1.16B}, [x0], #16 subs w2, w2, #1 - eor v0.16b, v0.16b, v1.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V1.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + st1 {V0.16B}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_192 ENDIF b L_aes_cbc_encrypt_arm64_crypto_done ; AES_CBC_256 L_aes_cbc_encrypt_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d, v29.2d}, [x4], #32 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D, V29.2D}, [x4], #32 L_aes_cbc_encrypt_arm64_crypto_loop_256 - ld1 {v30.2d}, [x4] - ld1 {v1.16b}, [x0], #16 + ld1 {V30.2D}, [x4] + ld1 {V1.16B}, [x0], #16 subs w2, w2, #1 - eor v0.16b, v0.16b, v1.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V1.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + st1 {V0.16B}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_256 ENDIF b L_aes_cbc_encrypt_arm64_crypto_done ; AES_CBC_128 L_aes_cbc_encrypt_arm64_crypto_start_128 IF :LNOT::DEF:NO_AES_128 - ld1 {v24.2d, v25.2d}, [x4], #32 + ld1 {V24.2D, V25.2D}, [x4], #32 L_aes_cbc_encrypt_arm64_crypto_loop_128 - ld1 {v26.2d}, [x4] - ld1 {v1.16b}, [x0], #16 + ld1 {V26.2D}, [x4] + ld1 {V1.16B}, [x0], #16 subs w2, w2, #1 - eor v0.16b, v0.16b, v1.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V1.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + st1 {V0.16B}, [x1], #16 bne L_aes_cbc_encrypt_arm64_crypto_loop_128 ENDIF L_aes_cbc_encrypt_arm64_crypto_done - st1 {v0.2d}, [x3] + st1 {V0.2D}, [x3] ret ENDP IF :DEF:HAVE_AES_DECRYPT @@ -3199,83 +3199,83 @@ L_aes_cbc_encrypt_arm64_crypto_done ALIGN 4 EXPORT AES_CBC_decrypt_AARCH64 AES_CBC_decrypt_AARCH64 PROC - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v0.2d}, [x3] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V0.2D}, [x3] lsr w2, w2, #4 cmp w5, #12 blt L_aes_cbc_decrypt_blocks_arm64_crypto_start_128 bgt L_aes_cbc_decrypt_blocks_arm64_crypto_start_256 ; AES_CBC_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D}, [x4] cmp w2, #10 ble L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1_long - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - mov v2.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v0.16b - mov v0.16b, v2.16b - st1 {v1.16b}, [x1], #16 + mov V2.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V0.16B + mov V0.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - eor v2.16b, v0.16b, v28.16b - mov v0.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v2.16b - st1 {v1.16b}, [x1], #16 + eor V2.16B, V0.16B, V28.16B + mov V0.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 ENDIF @@ -3283,83 +3283,83 @@ L_aes_cbc_decrypt_blocks_arm64_crypto_192_start_1 ; AES_CBC_256 L_aes_cbc_decrypt_blocks_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d, v29.2d}, [x4], #32 - ld1 {v30.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D, V29.2D}, [x4], #32 + ld1 {V30.2D}, [x4] cmp w2, #5 ble L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1_long - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - mov v2.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v1.16b, v1.16b, v0.16b - mov v0.16b, v2.16b - st1 {v1.16b}, [x1], #16 + mov V2.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V1.16B, V1.16B, V0.16B + mov V0.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - eor v2.16b, v0.16b, v30.16b - mov v0.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v2.16b - st1 {v1.16b}, [x1], #16 + eor V2.16B, V0.16B, V30.16B + mov V0.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 ENDIF @@ -3367,71 +3367,71 @@ L_aes_cbc_decrypt_blocks_arm64_crypto_256_start_1 ; AES_CBC_128 L_aes_cbc_decrypt_blocks_arm64_crypto_start_128 IF :LNOT::DEF:NO_AES_128 - ld1 {v24.2d, v25.2d}, [x4], #32 - ld1 {v26.2d}, [x4] + ld1 {V24.2D, V25.2D}, [x4], #32 + ld1 {V26.2D}, [x4] cmp w2, #24 ble L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1 L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1_long - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - mov v2.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v1.16b, v1.16b, v0.16b - mov v0.16b, v2.16b - st1 {v1.16b}, [x1], #16 + mov V2.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V1.16B, V1.16B, V0.16B + mov V0.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1_long b L_aes_cbc_decrypt_blocks_arm64_crypto_done L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1 - ld1 {v1.16b}, [x0], #16 + ld1 {V1.16B}, [x0], #16 sub w2, w2, #1 - eor v2.16b, v0.16b, v26.16b - mov v0.16b, v1.16b - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v2.16b - st1 {v1.16b}, [x1], #16 + eor V2.16B, V0.16B, V26.16B + mov V0.16B, V1.16B + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V2.16B + st1 {V1.16B}, [x1], #16 cmp w2, #1 bge L_aes_cbc_decrypt_blocks_arm64_crypto_128_start_1 ENDIF L_aes_cbc_decrypt_blocks_arm64_crypto_done - st1 {v0.2d}, [x3] + st1 {V0.2D}, [x3] ret ENDP ENDIF @@ -3447,25 +3447,25 @@ AES_CTR_encrypt_AARCH64 PROC stp x20, x21, [x29, #32] stp x22, x23, [x29, #48] stp x24, x25, [x29, #64] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x4], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x4], #0x40 - ld1 {v15.2d}, [x3] - rev64 v16.16b, v15.16b + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x4], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x4], #0x40 + ld1 {V15.2D}, [x3] + rev64 V16.16B, V15.16B lsr w8, w2, #4 and w2, w2, #15 - mov x9, v16.d[1] - mov x10, v16.d[0] + mov x9, V16.D[1] + mov x10, V16.D[0] cmp w7, #12 blt L_aes_ctr_encrypt_arm64_crypto_start_128 bgt L_aes_ctr_encrypt_arm64_crypto_start_256 ; AES_CTR_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40 - ld1 {v12.2d}, [x4] + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x4], #0x40 + ld1 {V12.2D}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_192_start_1 adds x11, x9, #1 @@ -3477,514 +3477,514 @@ AES_CTR_encrypt_AARCH64 PROC adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_192_start_8 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 - mov v17.d[0], x12 - mov v17.d[1], x11 - mov v18.d[0], x14 - mov v18.d[1], x13 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x0], #0x40 + mov V17.D[0], x12 + mov V17.D[1], x11 + mov V18.D[0], x14 + mov V18.D[1], x13 adds x17, x9, #4 - mov v19.d[0], x16 + mov V19.D[0], x16 adc x19, x10, xzr - mov v19.d[1], x15 + mov V19.D[1], x15 adds x20, x9, #5 - mov v20.d[0], x19 + mov V20.D[0], x19 adc x21, x10, xzr - mov v20.d[1], x17 + mov V20.D[1], x17 adds x22, x9, #6 - mov v21.d[0], x21 + mov V21.D[0], x21 adc x23, x10, xzr - mov v21.d[1], x20 + mov V21.D[1], x20 adds x24, x9, #7 - mov v22.d[0], x23 + mov V22.D[0], x23 adc x25, x10, xzr - mov v22.d[1], x22 - mov v23.d[0], x25 - mov v23.d[1], x24 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - rev64 v20.16b, v20.16b - rev64 v21.16b, v21.16b - rev64 v22.16b, v22.16b - rev64 v23.16b, v23.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v20.16b, v0.16b - aesmc v20.16b, v20.16b - aese v21.16b, v0.16b - aesmc v21.16b, v21.16b - aese v22.16b, v0.16b - aesmc v22.16b, v22.16b - aese v23.16b, v0.16b - aesmc v23.16b, v23.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v20.16b, v1.16b - aesmc v20.16b, v20.16b - aese v21.16b, v1.16b - aesmc v21.16b, v21.16b - aese v22.16b, v1.16b - aesmc v22.16b, v22.16b - aese v23.16b, v1.16b - aesmc v23.16b, v23.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v20.16b, v2.16b - aesmc v20.16b, v20.16b - aese v21.16b, v2.16b - aesmc v21.16b, v21.16b - aese v22.16b, v2.16b - aesmc v22.16b, v22.16b - aese v23.16b, v2.16b - aesmc v23.16b, v23.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v20.16b, v3.16b - aesmc v20.16b, v20.16b - aese v21.16b, v3.16b - aesmc v21.16b, v21.16b - aese v22.16b, v3.16b - aesmc v22.16b, v22.16b - aese v23.16b, v3.16b - aesmc v23.16b, v23.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v20.16b, v4.16b - aesmc v20.16b, v20.16b - aese v21.16b, v4.16b - aesmc v21.16b, v21.16b - aese v22.16b, v4.16b - aesmc v22.16b, v22.16b - aese v23.16b, v4.16b - aesmc v23.16b, v23.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v20.16b, v5.16b - aesmc v20.16b, v20.16b - aese v21.16b, v5.16b - aesmc v21.16b, v21.16b - aese v22.16b, v5.16b - aesmc v22.16b, v22.16b - aese v23.16b, v5.16b - aesmc v23.16b, v23.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v20.16b, v6.16b - aesmc v20.16b, v20.16b - aese v21.16b, v6.16b - aesmc v21.16b, v21.16b - aese v22.16b, v6.16b - aesmc v22.16b, v22.16b - aese v23.16b, v6.16b - aesmc v23.16b, v23.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v20.16b, v7.16b - aesmc v20.16b, v20.16b - aese v21.16b, v7.16b - aesmc v21.16b, v21.16b - aese v22.16b, v7.16b - aesmc v22.16b, v22.16b - aese v23.16b, v7.16b - aesmc v23.16b, v23.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v20.16b, v8.16b - aesmc v20.16b, v20.16b - aese v21.16b, v8.16b - aesmc v21.16b, v21.16b - aese v22.16b, v8.16b - aesmc v22.16b, v22.16b - aese v23.16b, v8.16b - aesmc v23.16b, v23.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v18.16b, v9.16b - aesmc v18.16b, v18.16b - aese v19.16b, v9.16b - aesmc v19.16b, v19.16b - aese v20.16b, v9.16b - aesmc v20.16b, v20.16b - aese v21.16b, v9.16b - aesmc v21.16b, v21.16b - aese v22.16b, v9.16b - aesmc v22.16b, v22.16b - aese v23.16b, v9.16b - aesmc v23.16b, v23.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v18.16b, v10.16b - aesmc v18.16b, v18.16b - aese v19.16b, v10.16b - aesmc v19.16b, v19.16b - aese v20.16b, v10.16b - aesmc v20.16b, v20.16b - aese v21.16b, v10.16b - aesmc v21.16b, v21.16b - aese v22.16b, v10.16b - aesmc v22.16b, v22.16b - aese v23.16b, v10.16b - aesmc v23.16b, v23.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - aese v18.16b, v11.16b - eor v18.16b, v18.16b, v12.16b - aese v19.16b, v11.16b - eor v19.16b, v19.16b, v12.16b - aese v20.16b, v11.16b - eor v20.16b, v20.16b, v12.16b - aese v21.16b, v11.16b - eor v21.16b, v21.16b, v12.16b - aese v22.16b, v11.16b - eor v22.16b, v22.16b, v12.16b - aese v23.16b, v11.16b - eor v23.16b, v23.16b, v12.16b + mov V22.D[1], x22 + mov V23.D[0], x25 + mov V23.D[1], x24 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + rev64 V20.16B, V20.16B + rev64 V21.16B, V21.16B + rev64 V22.16B, V22.16B + rev64 V23.16B, V23.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V20.16B, V0.16B + aesmc V20.16B, V20.16B + aese V21.16B, V0.16B + aesmc V21.16B, V21.16B + aese V22.16B, V0.16B + aesmc V22.16B, V22.16B + aese V23.16B, V0.16B + aesmc V23.16B, V23.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V20.16B, V1.16B + aesmc V20.16B, V20.16B + aese V21.16B, V1.16B + aesmc V21.16B, V21.16B + aese V22.16B, V1.16B + aesmc V22.16B, V22.16B + aese V23.16B, V1.16B + aesmc V23.16B, V23.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V20.16B, V2.16B + aesmc V20.16B, V20.16B + aese V21.16B, V2.16B + aesmc V21.16B, V21.16B + aese V22.16B, V2.16B + aesmc V22.16B, V22.16B + aese V23.16B, V2.16B + aesmc V23.16B, V23.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V20.16B, V3.16B + aesmc V20.16B, V20.16B + aese V21.16B, V3.16B + aesmc V21.16B, V21.16B + aese V22.16B, V3.16B + aesmc V22.16B, V22.16B + aese V23.16B, V3.16B + aesmc V23.16B, V23.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V20.16B, V4.16B + aesmc V20.16B, V20.16B + aese V21.16B, V4.16B + aesmc V21.16B, V21.16B + aese V22.16B, V4.16B + aesmc V22.16B, V22.16B + aese V23.16B, V4.16B + aesmc V23.16B, V23.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V20.16B, V5.16B + aesmc V20.16B, V20.16B + aese V21.16B, V5.16B + aesmc V21.16B, V21.16B + aese V22.16B, V5.16B + aesmc V22.16B, V22.16B + aese V23.16B, V5.16B + aesmc V23.16B, V23.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V20.16B, V6.16B + aesmc V20.16B, V20.16B + aese V21.16B, V6.16B + aesmc V21.16B, V21.16B + aese V22.16B, V6.16B + aesmc V22.16B, V22.16B + aese V23.16B, V6.16B + aesmc V23.16B, V23.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V20.16B, V7.16B + aesmc V20.16B, V20.16B + aese V21.16B, V7.16B + aesmc V21.16B, V21.16B + aese V22.16B, V7.16B + aesmc V22.16B, V22.16B + aese V23.16B, V7.16B + aesmc V23.16B, V23.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V20.16B, V8.16B + aesmc V20.16B, V20.16B + aese V21.16B, V8.16B + aesmc V21.16B, V21.16B + aese V22.16B, V8.16B + aesmc V22.16B, V22.16B + aese V23.16B, V8.16B + aesmc V23.16B, V23.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V18.16B, V9.16B + aesmc V18.16B, V18.16B + aese V19.16B, V9.16B + aesmc V19.16B, V19.16B + aese V20.16B, V9.16B + aesmc V20.16B, V20.16B + aese V21.16B, V9.16B + aesmc V21.16B, V21.16B + aese V22.16B, V9.16B + aesmc V22.16B, V22.16B + aese V23.16B, V9.16B + aesmc V23.16B, V23.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V18.16B, V10.16B + aesmc V18.16B, V18.16B + aese V19.16B, V10.16B + aesmc V19.16B, V19.16B + aese V20.16B, V10.16B + aesmc V20.16B, V20.16B + aese V21.16B, V10.16B + aesmc V21.16B, V21.16B + aese V22.16B, V10.16B + aesmc V22.16B, V22.16B + aese V23.16B, V10.16B + aesmc V23.16B, V23.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + aese V18.16B, V11.16B + eor V18.16B, V18.16B, V12.16B + aese V19.16B, V11.16B + eor V19.16B, V19.16B, V12.16B + aese V20.16B, V11.16B + eor V20.16B, V20.16B, V12.16B + aese V21.16B, V11.16B + eor V21.16B, V21.16B, V12.16B + aese V22.16B, V11.16B + eor V22.16B, V22.16B, V12.16B + aese V23.16B, V11.16B + eor V23.16B, V23.16B, V12.16B adds x9, x9, #8 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #8 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B adds x13, x13, #8 - eor v28.16b, v28.16b, v20.16b + eor V28.16B, V28.16B, V20.16B adc x14, x14, xzr - eor v29.16b, v29.16b, v21.16b + eor V29.16B, V29.16B, V21.16B adds x15, x15, #8 - eor v30.16b, v30.16b, v22.16b + eor V30.16B, V30.16B, V22.16B adc x16, x16, xzr - eor v31.16b, v31.16b, v23.16b + eor V31.16B, V31.16B, V23.16B sub w8, w8, #8 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + st1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_192_start_8 L_aes_ctr_encrypt_arm64_crypto_192_start_4 cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_192_start_2 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 adds x13, x9, #2 - mov v17.d[0], x12 + mov V17.D[0], x12 adc x14, x10, xzr - mov v17.d[1], x11 + mov V17.D[1], x11 adds x15, x9, #3 - mov v18.d[0], x14 + mov V18.D[0], x14 adc x16, x10, xzr - mov v18.d[1], x13 - mov v19.d[0], x16 - mov v19.d[1], x15 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v18.16b, v9.16b - aesmc v18.16b, v18.16b - aese v19.16b, v9.16b - aesmc v19.16b, v19.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v18.16b, v10.16b - aesmc v18.16b, v18.16b - aese v19.16b, v10.16b - aesmc v19.16b, v19.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - aese v18.16b, v11.16b - eor v18.16b, v18.16b, v12.16b - aese v19.16b, v11.16b - eor v19.16b, v19.16b, v12.16b + mov V18.D[1], x13 + mov V19.D[0], x16 + mov V19.D[1], x15 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V18.16B, V9.16B + aesmc V18.16B, V18.16B + aese V19.16B, V9.16B + aesmc V19.16B, V19.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V18.16B, V10.16B + aesmc V18.16B, V18.16B + aese V19.16B, V10.16B + aesmc V19.16B, V19.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + aese V18.16B, V11.16B + eor V18.16B, V18.16B, V12.16B + aese V19.16B, V11.16B + eor V19.16B, V19.16B, V12.16B adds x9, x9, #4 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #4 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B sub w8, w8, #4 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_192_start_2 cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_192_start_1 - ld1 {v24.16b, v25.16b}, [x0], #32 - eor v20.16b, v20.16b, v20.16b - ext v19.16b, v16.16b, v16.16b, #8 - movi v18.16b, #1 - ext v18.16b, v18.16b, v20.16b, #15 - add v17.2d, v19.2d, v18.2d - cmeq v19.2d, v17.2d, #0 - ext v19.16b, v20.16b, v19.16b, #8 - sub v17.2d, v17.2d, v19.2d - ext v17.16b, v17.16b, v17.16b, #8 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b + ld1 {V24.16B, V25.16B}, [x0], #32 + eor V20.16B, V20.16B, V20.16B + ext8 V19.16B, V16.16B, V16.16B, #8 + movi V18.16B, #1 + ext8 V18.16B, V18.16B, V20.16B, #15 + add V17.2D, V19.2D, V18.2D + cmeq V19.2D, V17.2D, #0 + ext8 V19.16B, V20.16B, V19.16B, #8 + sub V17.2D, V17.2D, V19.2D + ext8 V17.16B, V17.16B, V17.16B, #8 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B adds x9, x9, #2 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B sub w8, w8, #2 - st1 {v24.16b, v25.16b}, [x1], #32 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B}, [x1], #32 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_192_start_1 cbz w8, L_aes_ctr_encrypt_arm64_crypto_192_done - ld1 {v24.16b}, [x0], #16 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b + ld1 {V24.16B}, [x0], #16 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B adds x9, x9, #1 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - st1 {v24.16b}, [x1], #16 + st1 {V24.16B}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_192_done cbz w2, L_aes_ctr_encrypt_arm64_crypto_192_partial_done - mov v16.d[0], x10 - mov v16.d[1], x9 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b + mov V16.D[0], x10 + mov V16.D[1], x9 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B adds x9, x9, #1 adc x10, x10, xzr - st1 {v16.2d}, [x5] + st1 {V16.2D}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_192_start_byte @@ -4001,9 +4001,9 @@ L_aes_ctr_encrypt_arm64_crypto_192_partial_done ; AES_CTR_256 L_aes_ctr_encrypt_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40 - ld1 {v12.2d, v13.2d}, [x4], #32 - ld1 {v14.2d}, [x4] + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x4], #0x40 + ld1 {V12.2D, V13.2D}, [x4], #32 + ld1 {V14.2D}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_256_start_1 adds x11, x9, #1 @@ -4015,578 +4015,578 @@ L_aes_ctr_encrypt_arm64_crypto_start_256 adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_256_start_8 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 - mov v17.d[0], x12 - mov v17.d[1], x11 - mov v18.d[0], x14 - mov v18.d[1], x13 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x0], #0x40 + mov V17.D[0], x12 + mov V17.D[1], x11 + mov V18.D[0], x14 + mov V18.D[1], x13 adds x17, x9, #4 - mov v19.d[0], x16 + mov V19.D[0], x16 adc x19, x10, xzr - mov v19.d[1], x15 + mov V19.D[1], x15 adds x20, x9, #5 - mov v20.d[0], x19 + mov V20.D[0], x19 adc x21, x10, xzr - mov v20.d[1], x17 + mov V20.D[1], x17 adds x22, x9, #6 - mov v21.d[0], x21 + mov V21.D[0], x21 adc x23, x10, xzr - mov v21.d[1], x20 + mov V21.D[1], x20 adds x24, x9, #7 - mov v22.d[0], x23 + mov V22.D[0], x23 adc x25, x10, xzr - mov v22.d[1], x22 - mov v23.d[0], x25 - mov v23.d[1], x24 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - rev64 v20.16b, v20.16b - rev64 v21.16b, v21.16b - rev64 v22.16b, v22.16b - rev64 v23.16b, v23.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v20.16b, v0.16b - aesmc v20.16b, v20.16b - aese v21.16b, v0.16b - aesmc v21.16b, v21.16b - aese v22.16b, v0.16b - aesmc v22.16b, v22.16b - aese v23.16b, v0.16b - aesmc v23.16b, v23.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v20.16b, v1.16b - aesmc v20.16b, v20.16b - aese v21.16b, v1.16b - aesmc v21.16b, v21.16b - aese v22.16b, v1.16b - aesmc v22.16b, v22.16b - aese v23.16b, v1.16b - aesmc v23.16b, v23.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v20.16b, v2.16b - aesmc v20.16b, v20.16b - aese v21.16b, v2.16b - aesmc v21.16b, v21.16b - aese v22.16b, v2.16b - aesmc v22.16b, v22.16b - aese v23.16b, v2.16b - aesmc v23.16b, v23.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v20.16b, v3.16b - aesmc v20.16b, v20.16b - aese v21.16b, v3.16b - aesmc v21.16b, v21.16b - aese v22.16b, v3.16b - aesmc v22.16b, v22.16b - aese v23.16b, v3.16b - aesmc v23.16b, v23.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v20.16b, v4.16b - aesmc v20.16b, v20.16b - aese v21.16b, v4.16b - aesmc v21.16b, v21.16b - aese v22.16b, v4.16b - aesmc v22.16b, v22.16b - aese v23.16b, v4.16b - aesmc v23.16b, v23.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v20.16b, v5.16b - aesmc v20.16b, v20.16b - aese v21.16b, v5.16b - aesmc v21.16b, v21.16b - aese v22.16b, v5.16b - aesmc v22.16b, v22.16b - aese v23.16b, v5.16b - aesmc v23.16b, v23.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v20.16b, v6.16b - aesmc v20.16b, v20.16b - aese v21.16b, v6.16b - aesmc v21.16b, v21.16b - aese v22.16b, v6.16b - aesmc v22.16b, v22.16b - aese v23.16b, v6.16b - aesmc v23.16b, v23.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v20.16b, v7.16b - aesmc v20.16b, v20.16b - aese v21.16b, v7.16b - aesmc v21.16b, v21.16b - aese v22.16b, v7.16b - aesmc v22.16b, v22.16b - aese v23.16b, v7.16b - aesmc v23.16b, v23.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v20.16b, v8.16b - aesmc v20.16b, v20.16b - aese v21.16b, v8.16b - aesmc v21.16b, v21.16b - aese v22.16b, v8.16b - aesmc v22.16b, v22.16b - aese v23.16b, v8.16b - aesmc v23.16b, v23.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v18.16b, v9.16b - aesmc v18.16b, v18.16b - aese v19.16b, v9.16b - aesmc v19.16b, v19.16b - aese v20.16b, v9.16b - aesmc v20.16b, v20.16b - aese v21.16b, v9.16b - aesmc v21.16b, v21.16b - aese v22.16b, v9.16b - aesmc v22.16b, v22.16b - aese v23.16b, v9.16b - aesmc v23.16b, v23.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v18.16b, v10.16b - aesmc v18.16b, v18.16b - aese v19.16b, v10.16b - aesmc v19.16b, v19.16b - aese v20.16b, v10.16b - aesmc v20.16b, v20.16b - aese v21.16b, v10.16b - aesmc v21.16b, v21.16b - aese v22.16b, v10.16b - aesmc v22.16b, v22.16b - aese v23.16b, v10.16b - aesmc v23.16b, v23.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - aese v18.16b, v11.16b - aesmc v18.16b, v18.16b - aese v19.16b, v11.16b - aesmc v19.16b, v19.16b - aese v20.16b, v11.16b - aesmc v20.16b, v20.16b - aese v21.16b, v11.16b - aesmc v21.16b, v21.16b - aese v22.16b, v11.16b - aesmc v22.16b, v22.16b - aese v23.16b, v11.16b - aesmc v23.16b, v23.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v18.16b, v12.16b - aesmc v18.16b, v18.16b - aese v19.16b, v12.16b - aesmc v19.16b, v19.16b - aese v20.16b, v12.16b - aesmc v20.16b, v20.16b - aese v21.16b, v12.16b - aesmc v21.16b, v21.16b - aese v22.16b, v12.16b - aesmc v22.16b, v22.16b - aese v23.16b, v12.16b - aesmc v23.16b, v23.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v14.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v14.16b - aese v18.16b, v13.16b - eor v18.16b, v18.16b, v14.16b - aese v19.16b, v13.16b - eor v19.16b, v19.16b, v14.16b - aese v20.16b, v13.16b - eor v20.16b, v20.16b, v14.16b - aese v21.16b, v13.16b - eor v21.16b, v21.16b, v14.16b - aese v22.16b, v13.16b - eor v22.16b, v22.16b, v14.16b - aese v23.16b, v13.16b - eor v23.16b, v23.16b, v14.16b + mov V22.D[1], x22 + mov V23.D[0], x25 + mov V23.D[1], x24 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + rev64 V20.16B, V20.16B + rev64 V21.16B, V21.16B + rev64 V22.16B, V22.16B + rev64 V23.16B, V23.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V20.16B, V0.16B + aesmc V20.16B, V20.16B + aese V21.16B, V0.16B + aesmc V21.16B, V21.16B + aese V22.16B, V0.16B + aesmc V22.16B, V22.16B + aese V23.16B, V0.16B + aesmc V23.16B, V23.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V20.16B, V1.16B + aesmc V20.16B, V20.16B + aese V21.16B, V1.16B + aesmc V21.16B, V21.16B + aese V22.16B, V1.16B + aesmc V22.16B, V22.16B + aese V23.16B, V1.16B + aesmc V23.16B, V23.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V20.16B, V2.16B + aesmc V20.16B, V20.16B + aese V21.16B, V2.16B + aesmc V21.16B, V21.16B + aese V22.16B, V2.16B + aesmc V22.16B, V22.16B + aese V23.16B, V2.16B + aesmc V23.16B, V23.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V20.16B, V3.16B + aesmc V20.16B, V20.16B + aese V21.16B, V3.16B + aesmc V21.16B, V21.16B + aese V22.16B, V3.16B + aesmc V22.16B, V22.16B + aese V23.16B, V3.16B + aesmc V23.16B, V23.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V20.16B, V4.16B + aesmc V20.16B, V20.16B + aese V21.16B, V4.16B + aesmc V21.16B, V21.16B + aese V22.16B, V4.16B + aesmc V22.16B, V22.16B + aese V23.16B, V4.16B + aesmc V23.16B, V23.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V20.16B, V5.16B + aesmc V20.16B, V20.16B + aese V21.16B, V5.16B + aesmc V21.16B, V21.16B + aese V22.16B, V5.16B + aesmc V22.16B, V22.16B + aese V23.16B, V5.16B + aesmc V23.16B, V23.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V20.16B, V6.16B + aesmc V20.16B, V20.16B + aese V21.16B, V6.16B + aesmc V21.16B, V21.16B + aese V22.16B, V6.16B + aesmc V22.16B, V22.16B + aese V23.16B, V6.16B + aesmc V23.16B, V23.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V20.16B, V7.16B + aesmc V20.16B, V20.16B + aese V21.16B, V7.16B + aesmc V21.16B, V21.16B + aese V22.16B, V7.16B + aesmc V22.16B, V22.16B + aese V23.16B, V7.16B + aesmc V23.16B, V23.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V20.16B, V8.16B + aesmc V20.16B, V20.16B + aese V21.16B, V8.16B + aesmc V21.16B, V21.16B + aese V22.16B, V8.16B + aesmc V22.16B, V22.16B + aese V23.16B, V8.16B + aesmc V23.16B, V23.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V18.16B, V9.16B + aesmc V18.16B, V18.16B + aese V19.16B, V9.16B + aesmc V19.16B, V19.16B + aese V20.16B, V9.16B + aesmc V20.16B, V20.16B + aese V21.16B, V9.16B + aesmc V21.16B, V21.16B + aese V22.16B, V9.16B + aesmc V22.16B, V22.16B + aese V23.16B, V9.16B + aesmc V23.16B, V23.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V18.16B, V10.16B + aesmc V18.16B, V18.16B + aese V19.16B, V10.16B + aesmc V19.16B, V19.16B + aese V20.16B, V10.16B + aesmc V20.16B, V20.16B + aese V21.16B, V10.16B + aesmc V21.16B, V21.16B + aese V22.16B, V10.16B + aesmc V22.16B, V22.16B + aese V23.16B, V10.16B + aesmc V23.16B, V23.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + aese V18.16B, V11.16B + aesmc V18.16B, V18.16B + aese V19.16B, V11.16B + aesmc V19.16B, V19.16B + aese V20.16B, V11.16B + aesmc V20.16B, V20.16B + aese V21.16B, V11.16B + aesmc V21.16B, V21.16B + aese V22.16B, V11.16B + aesmc V22.16B, V22.16B + aese V23.16B, V11.16B + aesmc V23.16B, V23.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V18.16B, V12.16B + aesmc V18.16B, V18.16B + aese V19.16B, V12.16B + aesmc V19.16B, V19.16B + aese V20.16B, V12.16B + aesmc V20.16B, V20.16B + aese V21.16B, V12.16B + aesmc V21.16B, V21.16B + aese V22.16B, V12.16B + aesmc V22.16B, V22.16B + aese V23.16B, V12.16B + aesmc V23.16B, V23.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V14.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V14.16B + aese V18.16B, V13.16B + eor V18.16B, V18.16B, V14.16B + aese V19.16B, V13.16B + eor V19.16B, V19.16B, V14.16B + aese V20.16B, V13.16B + eor V20.16B, V20.16B, V14.16B + aese V21.16B, V13.16B + eor V21.16B, V21.16B, V14.16B + aese V22.16B, V13.16B + eor V22.16B, V22.16B, V14.16B + aese V23.16B, V13.16B + eor V23.16B, V23.16B, V14.16B adds x9, x9, #8 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #8 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B adds x13, x13, #8 - eor v28.16b, v28.16b, v20.16b + eor V28.16B, V28.16B, V20.16B adc x14, x14, xzr - eor v29.16b, v29.16b, v21.16b + eor V29.16B, V29.16B, V21.16B adds x15, x15, #8 - eor v30.16b, v30.16b, v22.16b + eor V30.16B, V30.16B, V22.16B adc x16, x16, xzr - eor v31.16b, v31.16b, v23.16b + eor V31.16B, V31.16B, V23.16B sub w8, w8, #8 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + st1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_256_start_8 L_aes_ctr_encrypt_arm64_crypto_256_start_4 cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_256_start_2 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 adds x13, x9, #2 - mov v17.d[0], x12 + mov V17.D[0], x12 adc x14, x10, xzr - mov v17.d[1], x11 + mov V17.D[1], x11 adds x15, x9, #3 - mov v18.d[0], x14 + mov V18.D[0], x14 adc x16, x10, xzr - mov v18.d[1], x13 - mov v19.d[0], x16 - mov v19.d[1], x15 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v18.16b, v9.16b - aesmc v18.16b, v18.16b - aese v19.16b, v9.16b - aesmc v19.16b, v19.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v18.16b, v10.16b - aesmc v18.16b, v18.16b - aese v19.16b, v10.16b - aesmc v19.16b, v19.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - aese v18.16b, v11.16b - aesmc v18.16b, v18.16b - aese v19.16b, v11.16b - aesmc v19.16b, v19.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v18.16b, v12.16b - aesmc v18.16b, v18.16b - aese v19.16b, v12.16b - aesmc v19.16b, v19.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v14.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v14.16b - aese v18.16b, v13.16b - eor v18.16b, v18.16b, v14.16b - aese v19.16b, v13.16b - eor v19.16b, v19.16b, v14.16b + mov V18.D[1], x13 + mov V19.D[0], x16 + mov V19.D[1], x15 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V18.16B, V9.16B + aesmc V18.16B, V18.16B + aese V19.16B, V9.16B + aesmc V19.16B, V19.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V18.16B, V10.16B + aesmc V18.16B, V18.16B + aese V19.16B, V10.16B + aesmc V19.16B, V19.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + aese V18.16B, V11.16B + aesmc V18.16B, V18.16B + aese V19.16B, V11.16B + aesmc V19.16B, V19.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V18.16B, V12.16B + aesmc V18.16B, V18.16B + aese V19.16B, V12.16B + aesmc V19.16B, V19.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V14.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V14.16B + aese V18.16B, V13.16B + eor V18.16B, V18.16B, V14.16B + aese V19.16B, V13.16B + eor V19.16B, V19.16B, V14.16B adds x9, x9, #4 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #4 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B sub w8, w8, #4 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_256_start_2 cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_256_start_1 - ld1 {v24.16b, v25.16b}, [x0], #32 - eor v20.16b, v20.16b, v20.16b - ext v19.16b, v16.16b, v16.16b, #8 - movi v18.16b, #1 - ext v18.16b, v18.16b, v20.16b, #15 - add v17.2d, v19.2d, v18.2d - cmeq v19.2d, v17.2d, #0 - ext v19.16b, v20.16b, v19.16b, #8 - sub v17.2d, v17.2d, v19.2d - ext v17.16b, v17.16b, v17.16b, #8 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v14.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v14.16b + ld1 {V24.16B, V25.16B}, [x0], #32 + eor V20.16B, V20.16B, V20.16B + ext8 V19.16B, V16.16B, V16.16B, #8 + movi V18.16B, #1 + ext8 V18.16B, V18.16B, V20.16B, #15 + add V17.2D, V19.2D, V18.2D + cmeq V19.2D, V17.2D, #0 + ext8 V19.16B, V20.16B, V19.16B, #8 + sub V17.2D, V17.2D, V19.2D + ext8 V17.16B, V17.16B, V17.16B, #8 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V14.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V14.16B adds x9, x9, #2 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B sub w8, w8, #2 - st1 {v24.16b, v25.16b}, [x1], #32 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B}, [x1], #32 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_256_start_1 cbz w8, L_aes_ctr_encrypt_arm64_crypto_256_done - ld1 {v24.16b}, [x0], #16 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v14.16b + ld1 {V24.16B}, [x0], #16 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V14.16B adds x9, x9, #1 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - st1 {v24.16b}, [x1], #16 + st1 {V24.16B}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_256_done cbz w2, L_aes_ctr_encrypt_arm64_crypto_256_partial_done - mov v16.d[0], x10 - mov v16.d[1], x9 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v14.16b + mov V16.D[0], x10 + mov V16.D[1], x9 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V14.16B adds x9, x9, #1 adc x10, x10, xzr - st1 {v16.2d}, [x5] + st1 {V16.2D}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_256_start_byte @@ -4603,8 +4603,8 @@ L_aes_ctr_encrypt_arm64_crypto_256_partial_done ; AES_CTR_128 L_aes_ctr_encrypt_arm64_crypto_start_128 IF :LNOT::DEF:NO_AES_128 - ld1 {v8.2d, v9.2d}, [x4], #32 - ld1 {v10.2d}, [x4] + ld1 {V8.2D, V9.2D}, [x4], #32 + ld1 {V10.2D}, [x4] cmp w8, #1 ble L_aes_ctr_encrypt_arm64_crypto_128_start_1 adds x11, x9, #1 @@ -4616,450 +4616,450 @@ L_aes_ctr_encrypt_arm64_crypto_start_128 adds x15, x9, #3 adc x16, x10, xzr L_aes_ctr_encrypt_arm64_crypto_128_start_8 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x0], #0x40 - mov v17.d[0], x12 - mov v17.d[1], x11 - mov v18.d[0], x14 - mov v18.d[1], x13 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x0], #0x40 + mov V17.D[0], x12 + mov V17.D[1], x11 + mov V18.D[0], x14 + mov V18.D[1], x13 adds x17, x9, #4 - mov v19.d[0], x16 + mov V19.D[0], x16 adc x19, x10, xzr - mov v19.d[1], x15 + mov V19.D[1], x15 adds x20, x9, #5 - mov v20.d[0], x19 + mov V20.D[0], x19 adc x21, x10, xzr - mov v20.d[1], x17 + mov V20.D[1], x17 adds x22, x9, #6 - mov v21.d[0], x21 + mov V21.D[0], x21 adc x23, x10, xzr - mov v21.d[1], x20 + mov V21.D[1], x20 adds x24, x9, #7 - mov v22.d[0], x23 + mov V22.D[0], x23 adc x25, x10, xzr - mov v22.d[1], x22 - mov v23.d[0], x25 - mov v23.d[1], x24 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - rev64 v20.16b, v20.16b - rev64 v21.16b, v21.16b - rev64 v22.16b, v22.16b - rev64 v23.16b, v23.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v20.16b, v0.16b - aesmc v20.16b, v20.16b - aese v21.16b, v0.16b - aesmc v21.16b, v21.16b - aese v22.16b, v0.16b - aesmc v22.16b, v22.16b - aese v23.16b, v0.16b - aesmc v23.16b, v23.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v20.16b, v1.16b - aesmc v20.16b, v20.16b - aese v21.16b, v1.16b - aesmc v21.16b, v21.16b - aese v22.16b, v1.16b - aesmc v22.16b, v22.16b - aese v23.16b, v1.16b - aesmc v23.16b, v23.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v20.16b, v2.16b - aesmc v20.16b, v20.16b - aese v21.16b, v2.16b - aesmc v21.16b, v21.16b - aese v22.16b, v2.16b - aesmc v22.16b, v22.16b - aese v23.16b, v2.16b - aesmc v23.16b, v23.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v20.16b, v3.16b - aesmc v20.16b, v20.16b - aese v21.16b, v3.16b - aesmc v21.16b, v21.16b - aese v22.16b, v3.16b - aesmc v22.16b, v22.16b - aese v23.16b, v3.16b - aesmc v23.16b, v23.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v20.16b, v4.16b - aesmc v20.16b, v20.16b - aese v21.16b, v4.16b - aesmc v21.16b, v21.16b - aese v22.16b, v4.16b - aesmc v22.16b, v22.16b - aese v23.16b, v4.16b - aesmc v23.16b, v23.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v20.16b, v5.16b - aesmc v20.16b, v20.16b - aese v21.16b, v5.16b - aesmc v21.16b, v21.16b - aese v22.16b, v5.16b - aesmc v22.16b, v22.16b - aese v23.16b, v5.16b - aesmc v23.16b, v23.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v20.16b, v6.16b - aesmc v20.16b, v20.16b - aese v21.16b, v6.16b - aesmc v21.16b, v21.16b - aese v22.16b, v6.16b - aesmc v22.16b, v22.16b - aese v23.16b, v6.16b - aesmc v23.16b, v23.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v20.16b, v7.16b - aesmc v20.16b, v20.16b - aese v21.16b, v7.16b - aesmc v21.16b, v21.16b - aese v22.16b, v7.16b - aesmc v22.16b, v22.16b - aese v23.16b, v7.16b - aesmc v23.16b, v23.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v20.16b, v8.16b - aesmc v20.16b, v20.16b - aese v21.16b, v8.16b - aesmc v21.16b, v21.16b - aese v22.16b, v8.16b - aesmc v22.16b, v22.16b - aese v23.16b, v8.16b - aesmc v23.16b, v23.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - aese v18.16b, v9.16b - eor v18.16b, v18.16b, v10.16b - aese v19.16b, v9.16b - eor v19.16b, v19.16b, v10.16b - aese v20.16b, v9.16b - eor v20.16b, v20.16b, v10.16b - aese v21.16b, v9.16b - eor v21.16b, v21.16b, v10.16b - aese v22.16b, v9.16b - eor v22.16b, v22.16b, v10.16b - aese v23.16b, v9.16b - eor v23.16b, v23.16b, v10.16b + mov V22.D[1], x22 + mov V23.D[0], x25 + mov V23.D[1], x24 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + rev64 V20.16B, V20.16B + rev64 V21.16B, V21.16B + rev64 V22.16B, V22.16B + rev64 V23.16B, V23.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V20.16B, V0.16B + aesmc V20.16B, V20.16B + aese V21.16B, V0.16B + aesmc V21.16B, V21.16B + aese V22.16B, V0.16B + aesmc V22.16B, V22.16B + aese V23.16B, V0.16B + aesmc V23.16B, V23.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V20.16B, V1.16B + aesmc V20.16B, V20.16B + aese V21.16B, V1.16B + aesmc V21.16B, V21.16B + aese V22.16B, V1.16B + aesmc V22.16B, V22.16B + aese V23.16B, V1.16B + aesmc V23.16B, V23.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V20.16B, V2.16B + aesmc V20.16B, V20.16B + aese V21.16B, V2.16B + aesmc V21.16B, V21.16B + aese V22.16B, V2.16B + aesmc V22.16B, V22.16B + aese V23.16B, V2.16B + aesmc V23.16B, V23.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V20.16B, V3.16B + aesmc V20.16B, V20.16B + aese V21.16B, V3.16B + aesmc V21.16B, V21.16B + aese V22.16B, V3.16B + aesmc V22.16B, V22.16B + aese V23.16B, V3.16B + aesmc V23.16B, V23.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V20.16B, V4.16B + aesmc V20.16B, V20.16B + aese V21.16B, V4.16B + aesmc V21.16B, V21.16B + aese V22.16B, V4.16B + aesmc V22.16B, V22.16B + aese V23.16B, V4.16B + aesmc V23.16B, V23.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V20.16B, V5.16B + aesmc V20.16B, V20.16B + aese V21.16B, V5.16B + aesmc V21.16B, V21.16B + aese V22.16B, V5.16B + aesmc V22.16B, V22.16B + aese V23.16B, V5.16B + aesmc V23.16B, V23.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V20.16B, V6.16B + aesmc V20.16B, V20.16B + aese V21.16B, V6.16B + aesmc V21.16B, V21.16B + aese V22.16B, V6.16B + aesmc V22.16B, V22.16B + aese V23.16B, V6.16B + aesmc V23.16B, V23.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V20.16B, V7.16B + aesmc V20.16B, V20.16B + aese V21.16B, V7.16B + aesmc V21.16B, V21.16B + aese V22.16B, V7.16B + aesmc V22.16B, V22.16B + aese V23.16B, V7.16B + aesmc V23.16B, V23.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V20.16B, V8.16B + aesmc V20.16B, V20.16B + aese V21.16B, V8.16B + aesmc V21.16B, V21.16B + aese V22.16B, V8.16B + aesmc V22.16B, V22.16B + aese V23.16B, V8.16B + aesmc V23.16B, V23.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + aese V18.16B, V9.16B + eor V18.16B, V18.16B, V10.16B + aese V19.16B, V9.16B + eor V19.16B, V19.16B, V10.16B + aese V20.16B, V9.16B + eor V20.16B, V20.16B, V10.16B + aese V21.16B, V9.16B + eor V21.16B, V21.16B, V10.16B + aese V22.16B, V9.16B + eor V22.16B, V22.16B, V10.16B + aese V23.16B, V9.16B + eor V23.16B, V23.16B, V10.16B adds x9, x9, #8 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #8 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B adds x13, x13, #8 - eor v28.16b, v28.16b, v20.16b + eor V28.16B, V28.16B, V20.16B adc x14, x14, xzr - eor v29.16b, v29.16b, v21.16b + eor V29.16B, V29.16B, V21.16B adds x15, x15, #8 - eor v30.16b, v30.16b, v22.16b + eor V30.16B, V30.16B, V22.16B adc x16, x16, xzr - eor v31.16b, v31.16b, v23.16b + eor V31.16B, V31.16B, V23.16B sub w8, w8, #8 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - st1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + st1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 cmp w8, #8 bge L_aes_ctr_encrypt_arm64_crypto_128_start_8 L_aes_ctr_encrypt_arm64_crypto_128_start_4 cmp w8, #4 blt L_aes_ctr_encrypt_arm64_crypto_128_start_2 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x0], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x0], #0x40 adds x13, x9, #2 - mov v17.d[0], x12 + mov V17.D[0], x12 adc x14, x10, xzr - mov v17.d[1], x11 + mov V17.D[1], x11 adds x15, x9, #3 - mov v18.d[0], x14 + mov V18.D[0], x14 adc x16, x10, xzr - mov v18.d[1], x13 - mov v19.d[0], x16 - mov v19.d[1], x15 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - rev64 v18.16b, v18.16b - rev64 v19.16b, v19.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v18.16b, v0.16b - aesmc v18.16b, v18.16b - aese v19.16b, v0.16b - aesmc v19.16b, v19.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v18.16b, v1.16b - aesmc v18.16b, v18.16b - aese v19.16b, v1.16b - aesmc v19.16b, v19.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v18.16b, v2.16b - aesmc v18.16b, v18.16b - aese v19.16b, v2.16b - aesmc v19.16b, v19.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v18.16b, v3.16b - aesmc v18.16b, v18.16b - aese v19.16b, v3.16b - aesmc v19.16b, v19.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v18.16b, v4.16b - aesmc v18.16b, v18.16b - aese v19.16b, v4.16b - aesmc v19.16b, v19.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v18.16b, v5.16b - aesmc v18.16b, v18.16b - aese v19.16b, v5.16b - aesmc v19.16b, v19.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v18.16b, v6.16b - aesmc v18.16b, v18.16b - aese v19.16b, v6.16b - aesmc v19.16b, v19.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v18.16b, v7.16b - aesmc v18.16b, v18.16b - aese v19.16b, v7.16b - aesmc v19.16b, v19.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v18.16b, v8.16b - aesmc v18.16b, v18.16b - aese v19.16b, v8.16b - aesmc v19.16b, v19.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - aese v18.16b, v9.16b - eor v18.16b, v18.16b, v10.16b - aese v19.16b, v9.16b - eor v19.16b, v19.16b, v10.16b + mov V18.D[1], x13 + mov V19.D[0], x16 + mov V19.D[1], x15 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + rev64 V18.16B, V18.16B + rev64 V19.16B, V19.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V18.16B, V0.16B + aesmc V18.16B, V18.16B + aese V19.16B, V0.16B + aesmc V19.16B, V19.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V18.16B, V1.16B + aesmc V18.16B, V18.16B + aese V19.16B, V1.16B + aesmc V19.16B, V19.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V18.16B, V2.16B + aesmc V18.16B, V18.16B + aese V19.16B, V2.16B + aesmc V19.16B, V19.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V18.16B, V3.16B + aesmc V18.16B, V18.16B + aese V19.16B, V3.16B + aesmc V19.16B, V19.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V18.16B, V4.16B + aesmc V18.16B, V18.16B + aese V19.16B, V4.16B + aesmc V19.16B, V19.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V18.16B, V5.16B + aesmc V18.16B, V18.16B + aese V19.16B, V5.16B + aesmc V19.16B, V19.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V18.16B, V6.16B + aesmc V18.16B, V18.16B + aese V19.16B, V6.16B + aesmc V19.16B, V19.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V18.16B, V7.16B + aesmc V18.16B, V18.16B + aese V19.16B, V7.16B + aesmc V19.16B, V19.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V18.16B, V8.16B + aesmc V18.16B, V18.16B + aese V19.16B, V8.16B + aesmc V19.16B, V19.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + aese V18.16B, V9.16B + eor V18.16B, V18.16B, V10.16B + aese V19.16B, V9.16B + eor V19.16B, V19.16B, V10.16B adds x9, x9, #4 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B adds x11, x11, #4 - eor v26.16b, v26.16b, v18.16b + eor V26.16B, V26.16B, V18.16B adc x12, x12, xzr - eor v27.16b, v27.16b, v19.16b + eor V27.16B, V27.16B, V19.16B sub w8, w8, #4 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_128_start_2 cmp w8, #2 blt L_aes_ctr_encrypt_arm64_crypto_128_start_1 - ld1 {v24.16b, v25.16b}, [x0], #32 - eor v20.16b, v20.16b, v20.16b - ext v19.16b, v16.16b, v16.16b, #8 - movi v18.16b, #1 - ext v18.16b, v18.16b, v20.16b, #15 - add v17.2d, v19.2d, v18.2d - cmeq v19.2d, v17.2d, #0 - ext v19.16b, v20.16b, v19.16b, #8 - sub v17.2d, v17.2d, v19.2d - ext v17.16b, v17.16b, v17.16b, #8 - rev64 v16.16b, v16.16b - rev64 v17.16b, v17.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b + ld1 {V24.16B, V25.16B}, [x0], #32 + eor V20.16B, V20.16B, V20.16B + ext8 V19.16B, V16.16B, V16.16B, #8 + movi V18.16B, #1 + ext8 V18.16B, V18.16B, V20.16B, #15 + add V17.2D, V19.2D, V18.2D + cmeq V19.2D, V17.2D, #0 + ext8 V19.16B, V20.16B, V19.16B, #8 + sub V17.2D, V17.2D, V19.2D + ext8 V17.16B, V17.16B, V17.16B, #8 + rev64 V16.16B, V16.16B + rev64 V17.16B, V17.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B adds x9, x9, #2 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - eor v25.16b, v25.16b, v17.16b + eor V25.16B, V25.16B, V17.16B sub w8, w8, #2 - st1 {v24.16b, v25.16b}, [x1], #32 - mov v16.d[0], x10 - mov v16.d[1], x9 + st1 {V24.16B, V25.16B}, [x1], #32 + mov V16.D[0], x10 + mov V16.D[1], x9 L_aes_ctr_encrypt_arm64_crypto_128_start_1 cbz w8, L_aes_ctr_encrypt_arm64_crypto_128_done - ld1 {v24.16b}, [x0], #16 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b + ld1 {V24.16B}, [x0], #16 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B adds x9, x9, #1 - eor v24.16b, v24.16b, v16.16b + eor V24.16B, V24.16B, V16.16B adc x10, x10, xzr - st1 {v24.16b}, [x1], #16 + st1 {V24.16B}, [x1], #16 L_aes_ctr_encrypt_arm64_crypto_128_done cbz w2, L_aes_ctr_encrypt_arm64_crypto_128_partial_done - mov v16.d[0], x10 - mov v16.d[1], x9 - rev64 v16.16b, v16.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b + mov V16.D[0], x10 + mov V16.D[1], x9 + rev64 V16.16B, V16.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B adds x9, x9, #1 adc x10, x10, xzr - st1 {v16.2d}, [x5] + st1 {V16.2D}, [x5] mov w13, #16 sub w13, w13, w2 L_aes_ctr_encrypt_arm64_crypto_128_start_byte @@ -5080,10 +5080,10 @@ L_aes_ctr_encrypt_arm64_crypto_done ldp x20, x21, [x29, #32] ldp x22, x23, [x29, #48] ldp x24, x25, [x29, #64] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -5093,48 +5093,48 @@ L_aes_ctr_encrypt_arm64_crypto_done ALIGN 4 EXPORT AES_GCM_set_key_AARCH64 AES_GCM_set_key_AARCH64 PROC - ld1 {v0.16b}, [x0] - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x1], #0x40 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b - aesmc v0.16b, v0.16b - aese v0.16b, v3.16b - aesmc v0.16b, v0.16b - aese v0.16b, v4.16b - aesmc v0.16b, v0.16b - ld1 {v1.2d, v2.2d, v3.2d, v4.2d}, [x1], #0x40 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b - aesmc v0.16b, v0.16b - aese v0.16b, v3.16b - aesmc v0.16b, v0.16b - aese v0.16b, v4.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B}, [x0] + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x1], #0x40 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B + aesmc V0.16B, V0.16B + aese V0.16B, V3.16B + aesmc V0.16B, V0.16B + aese V0.16B, V4.16B + aesmc V0.16B, V0.16B + ld1 {V1.2D, V2.2D, V3.2D, V4.2D}, [x1], #0x40 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B + aesmc V0.16B, V0.16B + aese V0.16B, V3.16B + aesmc V0.16B, V0.16B + aese V0.16B, V4.16B + aesmc V0.16B, V0.16B subs w3, w3, #10 - ld1 {v1.2d, v2.2d}, [x1], #32 - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x1], #32 + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B beq L_aes_gcm_set_key_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x1], #32 + ld1 {V1.2D, V2.2D}, [x1], #32 subs w3, w3, #2 - aesmc v0.16b, v0.16b - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + aesmc V0.16B, V0.16B + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B beq L_aes_gcm_set_key_arm64_crypto_round_done - ld1 {v1.2d, v2.2d}, [x1], #32 - aesmc v0.16b, v0.16b - aese v0.16b, v1.16b - aesmc v0.16b, v0.16b - aese v0.16b, v2.16b + ld1 {V1.2D, V2.2D}, [x1], #32 + aesmc V0.16B, V0.16B + aese V0.16B, V1.16B + aesmc V0.16B, V0.16B + aese V0.16B, V2.16B L_aes_gcm_set_key_arm64_crypto_round_done - ld1 {v1.2d}, [x1] - eor v0.16b, v0.16b, v1.16b - rbit v0.16b, v0.16b - st1 {v0.2d}, [x2] + ld1 {V1.2D}, [x1] + eor V0.16B, V0.16B, V1.16B + rbit V0.16B, V0.16B + st1 {V0.2D}, [x2] ret ENDP AREA |.text|, CODE, READONLY @@ -5147,20 +5147,20 @@ AES_GCM_encrypt_AARCH64 PROC stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] - movi v27.16b, #0x87 - eor v26.16b, v26.16b, v26.16b - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x10] + movi V27.16B, #0x87 + eor V26.16B, V26.16B, V26.16B + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 @@ -5168,14 +5168,14 @@ AES_GCM_encrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 @@ -5183,29 +5183,29 @@ AES_GCM_encrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w8, #0x400 csetm x16, lt @@ -5214,53 +5214,53 @@ AES_GCM_encrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_encrypt_arm64_crypto_h_done lsr w14, w8, #4 @@ -5271,102 +5271,102 @@ L_aes_gcm_encrypt_arm64_crypto_h_done cmp w14, #0x40 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_4 L_aes_gcm_encrypt_arm64_crypto_aad_start_8 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #8 cmp w14, #8 @@ -5377,57 +5377,57 @@ L_aes_gcm_encrypt_arm64_crypto_aad_start_8 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_aad_start_4 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #4 cmp w14, #4 @@ -5436,35 +5436,35 @@ L_aes_gcm_encrypt_arm64_crypto_aad_start_4 blt L_aes_gcm_encrypt_arm64_crypto_aad_done beq L_aes_gcm_encrypt_arm64_crypto_aad_start_1 L_aes_gcm_encrypt_arm64_crypto_aad_start_2 - ld1 {v18.16b, v19.16b}, [x7], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B, V19.16B}, [x7], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #2 cmp w14, #1 @@ -5473,33 +5473,33 @@ L_aes_gcm_encrypt_arm64_crypto_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_aad_start_1 cbz w14, L_aes_gcm_encrypt_arm64_crypto_aad_done L_aes_gcm_encrypt_arm64_crypto_aad_both_1 - ld1 {v18.16b}, [x7], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B}, [x7], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_aad_both_1 L_aes_gcm_encrypt_arm64_crypto_aad_done and w14, w8, #15 cbz w14, L_aes_gcm_encrypt_arm64_crypto_aad_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w14 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_aad_start_dw ldr x19, [x7], #8 @@ -5525,68 +5525,68 @@ L_aes_gcm_encrypt_arm64_crypto_aad_start_byte bne L_aes_gcm_encrypt_arm64_crypto_aad_start_byte L_aes_gcm_encrypt_arm64_crypto_aad_end_bytes sub x11, x11, x14 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_aad_partial_done ; Load Nonce cmp w4, #12 bne L_aes_gcm_encrypt_arm64_crypto_ghash_nonce ldr x16, [x3] - movi v13.4s, #1, lsl 24 + movi V13.4S, #1, lsl 24 ldr w17, [x3, #8] - mov v13.d[0], x16 - mov v13.s[2], w17 + mov V13.D[0], x16 + mov V13.S[2], w17 mov w15, #1 b L_aes_gcm_encrypt_arm64_crypto_done_nonce L_aes_gcm_encrypt_arm64_crypto_ghash_nonce - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B lsr w14, w4, #4 cbz w14, L_aes_gcm_encrypt_arm64_crypto_nonce_done L_aes_gcm_encrypt_arm64_crypto_nonce_start_1 - ld1 {v18.16b}, [x3], #16 - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.16B}, [x3], #16 + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_nonce_start_1 L_aes_gcm_encrypt_arm64_crypto_nonce_done and w24, w4, #15 cbz x24, L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w24 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_nonce_start_dw ldr x19, [x3], #8 @@ -5612,51 +5612,51 @@ L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte bne L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes sub x11, x11, x24 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done eor x14, x14, x14 ubfiz x24, x4, #3, #32 - mov v28.d[0], x14 - mov v28.d[1], x24 - rev64 v28.16b, v28.16b - rbit v28.16b, v28.16b - eor v13.16b, v13.16b, v28.16b - pmull v28.1q, v13.1d, v22.1d - pmull2 v29.1q, v13.2d, v22.2d - ext v31.16b, v13.16b, v13.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b - rbit v13.16b, v13.16b - mov w15, v13.s[3] + mov V28.D[0], x14 + mov V28.D[1], x24 + rev64 V28.16B, V28.16B + rbit V28.16B, V28.16B + eor V13.16B, V13.16B, V28.16B + pmull V28.1Q, V13.1D, V22.1D + pmull2 V29.1Q, V13.2D, V22.2D + ext8 V31.16B, V13.16B, V13.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B + rbit V13.16B, V13.16B + mov w15, V13.S[3] rev w15, w15 L_aes_gcm_encrypt_arm64_crypto_done_nonce - st1 {v13.2d}, [x12] + st1 {V13.2D}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_encrypt_arm64_crypto_start_128 @@ -5666,23 +5666,23 @@ L_aes_gcm_encrypt_arm64_crypto_done_nonce cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_192_start_4 L_aes_gcm_encrypt_arm64_crypto_192_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -5691,1202 +5691,1202 @@ L_aes_gcm_encrypt_arm64_crypto_192_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_end_8 L_aes_gcm_encrypt_arm64_crypto_192_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_192_both_8 L_aes_gcm_encrypt_arm64_crypto_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_192_done beq L_aes_gcm_encrypt_arm64_crypto_192_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_192_end_4 L_aes_gcm_encrypt_arm64_crypto_192_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_192_both_4 L_aes_gcm_encrypt_arm64_crypto_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_192_start_1 blt L_aes_gcm_encrypt_arm64_crypto_192_done L_aes_gcm_encrypt_arm64_crypto_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_192_done L_aes_gcm_encrypt_arm64_crypto_192_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_192_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_start_dw ldr x17, [x0], #8 @@ -6912,37 +6912,37 @@ L_aes_gcm_encrypt_arm64_crypto_192_start_byte bne L_aes_gcm_encrypt_arm64_crypto_192_start_byte L_aes_gcm_encrypt_arm64_crypto_192_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_out_start_dw @@ -6975,80 +6975,80 @@ L_aes_gcm_encrypt_arm64_crypto_192_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_192_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_192_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_192_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_dw ldr x16, [x11], #8 @@ -7081,23 +7081,23 @@ L_aes_gcm_encrypt_arm64_crypto_start_256 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_256_start_4 L_aes_gcm_encrypt_arm64_crypto_256_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -7106,1319 +7106,1319 @@ L_aes_gcm_encrypt_arm64_crypto_256_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_end_8 L_aes_gcm_encrypt_arm64_crypto_256_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_256_both_8 L_aes_gcm_encrypt_arm64_crypto_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9], #16 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_256_done beq L_aes_gcm_encrypt_arm64_crypto_256_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_256_end_4 L_aes_gcm_encrypt_arm64_crypto_256_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_256_both_4 L_aes_gcm_encrypt_arm64_crypto_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_256_start_1 blt L_aes_gcm_encrypt_arm64_crypto_256_done L_aes_gcm_encrypt_arm64_crypto_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_256_done L_aes_gcm_encrypt_arm64_crypto_256_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_256_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_start_dw ldr x17, [x0], #8 @@ -8444,43 +8444,43 @@ L_aes_gcm_encrypt_arm64_crypto_256_start_byte bne L_aes_gcm_encrypt_arm64_crypto_256_start_byte L_aes_gcm_encrypt_arm64_crypto_256_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_out_start_dw @@ -8513,88 +8513,88 @@ L_aes_gcm_encrypt_arm64_crypto_256_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_256_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v28.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V28.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - ldr q11, [x9, #-32] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - ldr q12, [x9, #-16] - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + ldr Q11, [x9, #-32] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + ldr Q12, [x9, #-16] + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_256_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_256_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_dw ldr x16, [x11], #8 @@ -8627,23 +8627,23 @@ L_aes_gcm_encrypt_arm64_crypto_start_128 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_128_start_4 L_aes_gcm_encrypt_arm64_crypto_128_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -8652,1090 +8652,1090 @@ L_aes_gcm_encrypt_arm64_crypto_128_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_end_8 L_aes_gcm_encrypt_arm64_crypto_128_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_128_both_8 L_aes_gcm_encrypt_arm64_crypto_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d}, [x9], #32 - ld1 {v10.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D}, [x9], #32 + ld1 {V10.2D}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_128_done beq L_aes_gcm_encrypt_arm64_crypto_128_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_128_end_4 L_aes_gcm_encrypt_arm64_crypto_128_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_128_both_4 L_aes_gcm_encrypt_arm64_crypto_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_128_start_1 blt L_aes_gcm_encrypt_arm64_crypto_128_done L_aes_gcm_encrypt_arm64_crypto_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_128_done L_aes_gcm_encrypt_arm64_crypto_128_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x0], #16 - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x0], #16 + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_128_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_start_dw ldr x17, [x0], #8 @@ -9761,33 +9761,33 @@ L_aes_gcm_encrypt_arm64_crypto_128_start_byte bne L_aes_gcm_encrypt_arm64_crypto_128_start_byte L_aes_gcm_encrypt_arm64_crypto_128_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_out_start_dw @@ -9820,76 +9820,76 @@ L_aes_gcm_encrypt_arm64_crypto_128_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_128_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - ext v31.16b, v26.16b, v26.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + ext8 V31.16B, V26.16B, V26.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_128_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_done L_aes_gcm_encrypt_arm64_crypto_128_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_dw ldr x16, [x11], #8 @@ -9920,10 +9920,10 @@ L_aes_gcm_encrypt_arm64_crypto_done ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -9938,20 +9938,20 @@ AES_GCM_decrypt_AARCH64 PROC stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] - movi v27.16b, #0x87 - eor v26.16b, v26.16b, v26.16b - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x10] + movi V27.16B, #0x87 + eor V26.16B, V26.16B, V26.16B + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 @@ -9959,14 +9959,14 @@ AES_GCM_decrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 @@ -9974,29 +9974,29 @@ AES_GCM_decrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w8, #0x400 csetm x16, lt @@ -10005,53 +10005,53 @@ AES_GCM_decrypt_AARCH64 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_decrypt_arm64_crypto_h_done lsr w14, w8, #4 @@ -10062,102 +10062,102 @@ L_aes_gcm_decrypt_arm64_crypto_h_done cmp w14, #0x40 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_4 L_aes_gcm_decrypt_arm64_crypto_aad_start_8 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #8 cmp w14, #8 @@ -10168,57 +10168,57 @@ L_aes_gcm_decrypt_arm64_crypto_aad_start_8 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_aad_start_4 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #4 cmp w14, #4 @@ -10227,35 +10227,35 @@ L_aes_gcm_decrypt_arm64_crypto_aad_start_4 blt L_aes_gcm_decrypt_arm64_crypto_aad_done beq L_aes_gcm_decrypt_arm64_crypto_aad_start_1 L_aes_gcm_decrypt_arm64_crypto_aad_start_2 - ld1 {v18.16b, v19.16b}, [x7], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B, V19.16B}, [x7], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #2 cmp w14, #1 @@ -10264,33 +10264,33 @@ L_aes_gcm_decrypt_arm64_crypto_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_aad_start_1 cbz w14, L_aes_gcm_decrypt_arm64_crypto_aad_done L_aes_gcm_decrypt_arm64_crypto_aad_both_1 - ld1 {v18.16b}, [x7], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B}, [x7], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_aad_both_1 L_aes_gcm_decrypt_arm64_crypto_aad_done and w14, w8, #15 cbz w14, L_aes_gcm_decrypt_arm64_crypto_aad_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w14 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_aad_start_dw ldr x19, [x7], #8 @@ -10316,68 +10316,68 @@ L_aes_gcm_decrypt_arm64_crypto_aad_start_byte bne L_aes_gcm_decrypt_arm64_crypto_aad_start_byte L_aes_gcm_decrypt_arm64_crypto_aad_end_bytes sub x11, x11, x14 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_aad_partial_done ; Load Nonce cmp w4, #12 bne L_aes_gcm_decrypt_arm64_crypto_ghash_nonce ldr x16, [x3] - movi v13.4s, #1, lsl 24 + movi V13.4S, #1, lsl 24 ldr w17, [x3, #8] - mov v13.d[0], x16 - mov v13.s[2], w17 + mov V13.D[0], x16 + mov V13.S[2], w17 mov w15, #1 b L_aes_gcm_decrypt_arm64_crypto_done_nonce L_aes_gcm_decrypt_arm64_crypto_ghash_nonce - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B lsr w14, w4, #4 cbz w14, L_aes_gcm_decrypt_arm64_crypto_nonce_done L_aes_gcm_decrypt_arm64_crypto_nonce_start_1 - ld1 {v18.16b}, [x3], #16 - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.16B}, [x3], #16 + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_nonce_start_1 L_aes_gcm_decrypt_arm64_crypto_nonce_done and w24, w4, #15 cbz x24, L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w24 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_nonce_start_dw ldr x19, [x3], #8 @@ -10403,51 +10403,51 @@ L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte bne L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes sub x11, x11, x24 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done eor x14, x14, x14 ubfiz x24, x4, #3, #32 - mov v28.d[0], x14 - mov v28.d[1], x24 - rev64 v28.16b, v28.16b - rbit v28.16b, v28.16b - eor v13.16b, v13.16b, v28.16b - pmull v28.1q, v13.1d, v22.1d - pmull2 v29.1q, v13.2d, v22.2d - ext v31.16b, v13.16b, v13.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b - rbit v13.16b, v13.16b - mov w15, v13.s[3] + mov V28.D[0], x14 + mov V28.D[1], x24 + rev64 V28.16B, V28.16B + rbit V28.16B, V28.16B + eor V13.16B, V13.16B, V28.16B + pmull V28.1Q, V13.1D, V22.1D + pmull2 V29.1Q, V13.2D, V22.2D + ext8 V31.16B, V13.16B, V13.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B + rbit V13.16B, V13.16B + mov w15, V13.S[3] rev w15, w15 L_aes_gcm_decrypt_arm64_crypto_done_nonce - st1 {v13.2d}, [x12] + st1 {V13.2D}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_decrypt_arm64_crypto_start_128 @@ -10457,1228 +10457,1228 @@ L_aes_gcm_decrypt_arm64_crypto_done_nonce cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_192_start_4 L_aes_gcm_decrypt_arm64_crypto_192_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_end_8 L_aes_gcm_decrypt_arm64_crypto_192_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_192_both_8 L_aes_gcm_decrypt_arm64_crypto_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_192_done beq L_aes_gcm_decrypt_arm64_crypto_192_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_192_end_4 L_aes_gcm_decrypt_arm64_crypto_192_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_192_both_4 L_aes_gcm_decrypt_arm64_crypto_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_192_start_1 blt L_aes_gcm_decrypt_arm64_crypto_192_done L_aes_gcm_decrypt_arm64_crypto_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_192_done L_aes_gcm_decrypt_arm64_crypto_192_start_1 - ld1 {v15.16b}, [x0], #16 + ld1 {V15.16B}, [x0], #16 add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.16b}, [x1], #16 + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.16B}, [x1], #16 L_aes_gcm_decrypt_arm64_crypto_192_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_192_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_start_dw ldr x17, [x0], #8 @@ -11704,56 +11704,56 @@ L_aes_gcm_decrypt_arm64_crypto_192_start_byte bne L_aes_gcm_decrypt_arm64_crypto_192_start_byte L_aes_gcm_decrypt_arm64_crypto_192_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_out_start_dw ldr x17, [x11], #8 @@ -11779,63 +11779,63 @@ L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_192_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_192_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_192_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_dw ldr x16, [x5], #8 @@ -11861,9 +11861,9 @@ L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_192_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte @@ -11871,11 +11871,11 @@ L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -11889,1344 +11889,1344 @@ L_aes_gcm_decrypt_arm64_crypto_start_256 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_256_start_4 L_aes_gcm_decrypt_arm64_crypto_256_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_end_8 L_aes_gcm_decrypt_arm64_crypto_256_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_256_both_8 L_aes_gcm_decrypt_arm64_crypto_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9], #16 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_256_done beq L_aes_gcm_decrypt_arm64_crypto_256_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_256_end_4 L_aes_gcm_decrypt_arm64_crypto_256_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_256_both_4 L_aes_gcm_decrypt_arm64_crypto_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_256_start_1 blt L_aes_gcm_decrypt_arm64_crypto_256_done L_aes_gcm_decrypt_arm64_crypto_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_256_done L_aes_gcm_decrypt_arm64_crypto_256_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_256_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_256_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_start_dw ldr x17, [x0], #8 @@ -13252,62 +13252,62 @@ L_aes_gcm_decrypt_arm64_crypto_256_start_byte bne L_aes_gcm_decrypt_arm64_crypto_256_start_byte L_aes_gcm_decrypt_arm64_crypto_256_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_out_start_dw ldr x17, [x11], #8 @@ -13333,71 +13333,71 @@ L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_256_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v28.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V28.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - ldr q11, [x9, #-32] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - ldr q12, [x9, #-16] - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + ldr Q11, [x9, #-32] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + ldr Q12, [x9, #-16] + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_256_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_256_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_dw ldr x16, [x5], #8 @@ -13423,9 +13423,9 @@ L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_256_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte @@ -13433,11 +13433,11 @@ L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -13451,1115 +13451,1115 @@ L_aes_gcm_decrypt_arm64_crypto_start_128 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_128_start_4 L_aes_gcm_decrypt_arm64_crypto_128_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_end_8 L_aes_gcm_decrypt_arm64_crypto_128_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_128_both_8 L_aes_gcm_decrypt_arm64_crypto_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d}, [x9], #32 - ld1 {v10.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D}, [x9], #32 + ld1 {V10.2D}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_128_done beq L_aes_gcm_decrypt_arm64_crypto_128_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_128_end_4 L_aes_gcm_decrypt_arm64_crypto_128_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_128_both_4 L_aes_gcm_decrypt_arm64_crypto_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_128_start_1 blt L_aes_gcm_decrypt_arm64_crypto_128_done L_aes_gcm_decrypt_arm64_crypto_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_128_done L_aes_gcm_decrypt_arm64_crypto_128_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x0], #16 - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x0], #16 + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_128_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_128_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_start_dw ldr x17, [x0], #8 @@ -14585,52 +14585,52 @@ L_aes_gcm_decrypt_arm64_crypto_128_start_byte bne L_aes_gcm_decrypt_arm64_crypto_128_start_byte L_aes_gcm_decrypt_arm64_crypto_128_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_out_start_dw ldr x17, [x11], #8 @@ -14656,59 +14656,59 @@ L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_128_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - ext v31.16b, v26.16b, v26.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + ext8 V31.16B, V26.16B, V26.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_128_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_128_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_dw ldr x16, [x5], #8 @@ -14734,9 +14734,9 @@ L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_128_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte @@ -14744,11 +14744,11 @@ L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -14760,10 +14760,10 @@ L_aes_gcm_decrypt_arm64_crypto_done ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -14779,20 +14779,20 @@ AES_GCM_encrypt_AARCH64_EOR3 PROC stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] - movi v27.16b, #0x87 - eor v26.16b, v26.16b, v26.16b - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x10] + movi V27.16B, #0x87 + eor V26.16B, V26.16B, V26.16B + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 @@ -14800,14 +14800,14 @@ AES_GCM_encrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 @@ -14815,28 +14815,28 @@ AES_GCM_encrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w8, #0x400 csetm x16, lt @@ -14845,51 +14845,51 @@ AES_GCM_encrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_encrypt_arm64_crypto_eor3_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_encrypt_arm64_crypto_eor3_h_done lsr w14, w8, #4 @@ -14900,94 +14900,94 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_h_done cmp w14, #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_8 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #8 cmp w14, #8 @@ -14998,53 +14998,53 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_8 cmp w14, #16 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #4 cmp w14, #4 @@ -15053,33 +15053,33 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 - ld1 {v18.16b, v19.16b}, [x7], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B, V19.16B}, [x7], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #2 cmp w14, #1 @@ -15088,32 +15088,32 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1 cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done L_aes_gcm_encrypt_arm64_crypto_eor3_aad_both_1 - ld1 {v18.16b}, [x7], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B}, [x7], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_both_1 L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done and w14, w8, #15 cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w14 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_dw ldr x19, [x7], #8 @@ -15139,66 +15139,66 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte bne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_aad_end_bytes sub x11, x11, x14 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_aad_partial_done ; Load Nonce cmp w4, #12 bne L_aes_gcm_encrypt_arm64_crypto_eor3_ghash_nonce ldr x16, [x3] - movi v13.4s, #1, lsl 24 + movi V13.4S, #1, lsl 24 ldr w17, [x3, #8] - mov v13.d[0], x16 - mov v13.s[2], w17 + mov V13.D[0], x16 + mov V13.S[2], w17 mov w15, #1 b L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce L_aes_gcm_encrypt_arm64_crypto_eor3_ghash_nonce - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B lsr w14, w4, #4 cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_done L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_1 - ld1 {v18.16b}, [x3], #16 - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.16B}, [x3], #16 + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_1 L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_done and w24, w4, #15 cbz x24, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w24 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_dw ldr x19, [x3], #8 @@ -15224,49 +15224,49 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte bne L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes sub x11, x11, x24 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done eor x14, x14, x14 ubfiz x24, x4, #3, #32 - mov v28.d[0], x14 - mov v28.d[1], x24 - rev64 v28.16b, v28.16b - rbit v28.16b, v28.16b - eor v13.16b, v13.16b, v28.16b - pmull v28.1q, v13.1d, v22.1d - pmull2 v29.1q, v13.2d, v22.2d - ext v31.16b, v13.16b, v13.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b - rbit v13.16b, v13.16b - mov w15, v13.s[3] + mov V28.D[0], x14 + mov V28.D[1], x24 + rev64 V28.16B, V28.16B + rbit V28.16B, V28.16B + eor V13.16B, V13.16B, V28.16B + pmull V28.1Q, V13.1D, V22.1D + pmull2 V29.1Q, V13.2D, V22.2D + ext8 V31.16B, V13.16B, V13.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B + rbit V13.16B, V13.16B + mov w15, V13.S[3] rev w15, w15 L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce - st1 {v13.2d}, [x12] + st1 {V13.2D}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_encrypt_arm64_crypto_eor3_start_128 @@ -15276,23 +15276,23 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -15301,1175 +15301,1175 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_dw ldr x17, [x0], #8 @@ -16495,37 +16495,37 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_dw @@ -16558,78 +16558,78 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_dw ldr x16, [x11], #8 @@ -16662,23 +16662,23 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_start_256 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -16687,1292 +16687,1292 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9], #16 cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_dw ldr x17, [x0], #8 @@ -17998,43 +17998,43 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_dw @@ -18067,86 +18067,86 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v28.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V28.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - ldr q11, [x9, #-32] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - ldr q12, [x9, #-16] - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + ldr Q11, [x9, #-32] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + ldr Q12, [x9, #-16] + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_dw ldr x16, [x11], #8 @@ -18179,23 +18179,23 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_start_128 cmp w14, #32 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 rev w23, w23 rev w22, w22 @@ -18204,1063 +18204,1063 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_8 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v0.16b}, [x0], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v1.16b}, [x0], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v2.16b}, [x0], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - ld1 {v3.16b}, [x0], #16 - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x12] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V0.16B}, [x0], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V1.16B}, [x0], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V2.16B}, [x0], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + ld1 {V3.16B}, [x0], #16 + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x12] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w23, w23 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w22, w22 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w21, w21 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w20, w20 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w19, w19 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w17, w17 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w16, w15 - mov v14.s[3], w24 - mov v15.s[3], w23 - mov v16.s[3], w22 - mov v17.s[3], w21 - mov v8.s[3], w20 - mov v9.s[3], w19 - mov v10.s[3], w17 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w24 + mov V15.S[3], w23 + mov V16.S[3], w22 + mov V17.S[3], w21 + mov V8.S[3], w20 + mov V9.S[3], w19 + mov V10.S[3], w17 + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_8 L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d}, [x9], #32 - ld1 {v10.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D}, [x9], #32 + ld1 {V10.2D}, [x9] cmp w14, #1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_done beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1 cmp w14, #4 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 rev w19, w19 rev w17, w17 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x0], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - ld1 {v21.16b}, [x0], #16 - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x0], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + ld1 {V21.16B}, [x0], #16 + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 rev w19, w19 rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w19 - mov v16.s[3], w17 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w20 + mov V15.S[3], w19 + mov V16.S[3], w17 + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w14, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x1], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x1], #0x40 bge L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_4 L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 rev w16, w15 - mov v14.s[3], w20 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w20 + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x0], #16 - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x0], #16 + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_done ands w14, w2, #15 beq L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B mov w19, w14 - st1 {v16.2d}, [x11] + st1 {V16.2D}, [x11] cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_dw ldr x17, [x0], #8 @@ -19286,33 +19286,33 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_bytes sub x11, x11, x14 - ld1 {v16.2d}, [x11] + ld1 {V16.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v16.16b, v16.16b, v14.16b - st1 {v16.2d}, [x11] + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V16.16B, V16.16B, V14.16B + st1 {V16.2D}, [x11] mov w19, w14 cmp x19, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_dw @@ -19345,74 +19345,74 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero strb wzr, [x11], #1 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero sub x11, x11, #16 - ld1 {v14.2d}, [x11] - rbit v14.16b, v14.16b - eor v15.16b, v26.16b, v14.16b + ld1 {V14.2D}, [x11] + rbit V14.16B, V14.16B + eor V15.16B, V26.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v15.1d, v22.1d - pmull2 v29.1q, v15.2d, v22.2d - ext v31.16b, v15.16b, v15.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V15.1D, V22.1D + pmull2 V29.1Q, V15.2D, V22.2D + ext8 V31.16B, V15.16B, V15.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - ext v31.16b, v26.16b, v26.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + ext8 V31.16B, V26.16B, V26.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 bne L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_partial - st1 {v26.16b}, [x5] + st1 {V26.16B}, [x5] b L_aes_gcm_encrypt_arm64_crypto_eor3_done L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_partial - st1 {v26.16b}, [x11] + st1 {V26.16B}, [x11] cmp w6, #8 blt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_dw ldr x16, [x11], #8 @@ -19443,10 +19443,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_done ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -19461,20 +19461,20 @@ AES_GCM_decrypt_AARCH64_EOR3 PROC stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] ldr x11, [x29, #168] ldr x12, [x29, #176] ldr w13, [x29, #184] - movi v27.16b, #0x87 - eor v26.16b, v26.16b, v26.16b - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x10] + movi V27.16B, #0x87 + eor V26.16B, V26.16B, V26.16B + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x10] cmp w8, #0x40 csetm x16, lt cmp w2, #32 @@ -19482,14 +19482,14 @@ AES_GCM_decrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w8, #0x100 csetm x16, lt cmp w2, #0x40 @@ -19497,28 +19497,28 @@ AES_GCM_decrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w8, #0x400 csetm x16, lt @@ -19527,51 +19527,51 @@ AES_GCM_decrypt_AARCH64_EOR3 PROC ands x16, x16, x17 bne L_aes_gcm_decrypt_arm64_crypto_eor3_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_decrypt_arm64_crypto_eor3_h_done lsr w14, w8, #4 @@ -19582,94 +19582,94 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_h_done cmp w14, #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_8 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #8 cmp w14, #8 @@ -19680,53 +19680,53 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_8 cmp w14, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x7], #0x40 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x7], #0x40 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #4 cmp w14, #4 @@ -19735,33 +19735,33 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 - ld1 {v18.16b, v19.16b}, [x7], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B, V19.16B}, [x7], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH sub w14, w14, #2 cmp w14, #1 @@ -19770,32 +19770,32 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_2 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1 cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done L_aes_gcm_decrypt_arm64_crypto_eor3_aad_both_1 - ld1 {v18.16b}, [x7], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.16B}, [x7], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_both_1 L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done and w14, w8, #15 cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w14 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_dw ldr x19, [x7], #8 @@ -19821,66 +19821,66 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_aad_end_bytes sub x11, x11, x14 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_aad_partial_done ; Load Nonce cmp w4, #12 bne L_aes_gcm_decrypt_arm64_crypto_eor3_ghash_nonce ldr x16, [x3] - movi v13.4s, #1, lsl 24 + movi V13.4S, #1, lsl 24 ldr w17, [x3, #8] - mov v13.d[0], x16 - mov v13.s[2], w17 + mov V13.D[0], x16 + mov V13.S[2], w17 mov w15, #1 b L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce L_aes_gcm_decrypt_arm64_crypto_eor3_ghash_nonce - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B lsr w14, w4, #4 cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_done L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_1 - ld1 {v18.16b}, [x3], #16 - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.16B}, [x3], #16 + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH subs w14, w14, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_1 L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_done and w24, w4, #15 cbz x24, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov w20, w24 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp w20, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_dw ldr x19, [x3], #8 @@ -19906,49 +19906,49 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes sub x11, x11, x24 - ld1 {v18.2d}, [x11] - rbit v18.16b, v18.16b - eor v21.16b, v13.16b, v18.16b + ld1 {V18.2D}, [x11] + rbit V18.16B, V18.16B + eor V21.16B, V13.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done eor x14, x14, x14 ubfiz x24, x4, #3, #32 - mov v28.d[0], x14 - mov v28.d[1], x24 - rev64 v28.16b, v28.16b - rbit v28.16b, v28.16b - eor v13.16b, v13.16b, v28.16b - pmull v28.1q, v13.1d, v22.1d - pmull2 v29.1q, v13.2d, v22.2d - ext v31.16b, v13.16b, v13.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v13.16b, v28.16b, v30.16b - rbit v13.16b, v13.16b - mov w15, v13.s[3] + mov V28.D[0], x14 + mov V28.D[1], x24 + rev64 V28.16B, V28.16B + rbit V28.16B, V28.16B + eor V13.16B, V13.16B, V28.16B + pmull V28.1Q, V13.1D, V22.1D + pmull2 V29.1Q, V13.2D, V22.2D + ext8 V31.16B, V13.16B, V13.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V13.16B, V28.16B, V30.16B + rbit V13.16B, V13.16B + mov w15, V13.S[3] rev w15, w15 L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce - st1 {v13.2d}, [x12] + st1 {V13.2D}, [x12] lsr w14, w2, #4 cmp w13, #12 blt L_aes_gcm_decrypt_arm64_crypto_eor3_start_128 @@ -19958,1201 +19958,1201 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1 - ld1 {v15.16b}, [x0], #16 + ld1 {V15.16B}, [x0], #16 add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.16b}, [x1], #16 + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.16B}, [x1], #16 L_aes_gcm_decrypt_arm64_crypto_eor3_192_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_dw ldr x17, [x0], #8 @@ -21178,55 +21178,55 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_dw ldr x17, [x11], #8 @@ -21252,62 +21252,62 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_dw ldr x16, [x5], #8 @@ -21333,9 +21333,9 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte @@ -21343,11 +21343,11 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -21361,1317 +21361,1317 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_start_256 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40 - ld1 {v12.2d}, [x9], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x9], #0x40 + ld1 {V12.2D}, [x9], #16 cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_256_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_dw ldr x17, [x0], #8 @@ -22697,61 +22697,61 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] ; Done GHASH - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_dw ldr x17, [x11], #8 @@ -22777,70 +22777,70 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v28.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v26.16b, v26.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V28.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V26.16B, V26.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - ldr q11, [x9, #-32] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - ldr q12, [x9, #-16] - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x9] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - ldr q30, [x9, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + ldr Q11, [x9, #-32] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + ldr Q12, [x9, #-16] + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x9] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + ldr Q30, [x9, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_dw ldr x16, [x5], #8 @@ -22866,9 +22866,9 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte @@ -22876,11 +22876,11 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -22894,1088 +22894,1088 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_start_128 cmp w14, #32 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w24, w24 - mov v14.s[3], w24 + mov V14.S[3], w24 rev w23, w23 - mov v15.s[3], w23 + mov V15.S[3], w23 rev w22, w22 - mov v16.s[3], w22 + mov V16.S[3], w22 rev w21, w21 - mov v17.s[3], w21 + mov V17.S[3], w21 rev w20, w20 - mov v8.s[3], w20 + mov V8.S[3], w20 rev w19, w19 - mov v9.s[3], w19 + mov V9.S[3], w19 rev w17, w17 - mov v10.s[3], w17 + mov V10.S[3], w17 rev w16, w15 - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - ld1 {v0.16b}, [x0], #16 - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v1.16b}, [x0], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v2.16b}, [x0], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v3.16b}, [x0], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x12] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + ld1 {V0.16B}, [x0], #16 + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V1.16B}, [x0], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V2.16B}, [x0], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V3.16B}, [x0], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x12] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_8 - ldr q12, [x9] + ldr Q12, [x9] add w24, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w23, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w22, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w21, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w20, w15, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w19, w15, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w17, w15, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w15, w15, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w24, w24 - rbit v19.16b, v19.16b - mov v14.s[3], w24 + rbit V19.16B, V19.16B + mov V14.S[3], w24 rev w23, w23 - rbit v20.16b, v20.16b - mov v15.s[3], w23 + rbit V20.16B, V20.16B + mov V15.S[3], w23 rev w22, w22 - rbit v21.16b, v21.16b - mov v16.s[3], w22 + rbit V21.16B, V21.16B + mov V16.S[3], w22 rev w21, w21 - rbit v0.16b, v0.16b - mov v17.s[3], w21 + rbit V0.16B, V0.16B + mov V17.S[3], w21 rev w20, w20 - rbit v1.16b, v1.16b - mov v8.s[3], w20 + rbit V1.16B, V1.16B + mov V8.S[3], w20 rev w19, w19 - rbit v2.16b, v2.16b - mov v9.s[3], w19 + rbit V2.16B, V2.16B + mov V9.S[3], w19 rev w17, w17 - rbit v3.16b, v3.16b - mov v10.s[3], w17 + rbit V3.16B, V3.16B + mov V10.S[3], w17 rev w16, w15 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w16 - ldr q13, [x9, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w16 + ldr Q13, [x9, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w14, w14, #8 - ldr q12, [x9, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x9, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x0], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x0], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x0], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x9, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x0], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x0], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x0], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x0], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x0], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x9, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x12] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x1], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x0], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x0], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x0], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x9, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x0], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x0], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x0], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x0], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x0], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x9, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x12] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x1], #0x40 cmp w14, #8 bge L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_8 L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40 - ld1 {v8.2d, v9.2d}, [x9], #32 - ld1 {v10.2d}, [x9] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x9], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x9], #0x40 + ld1 {V8.2D, V9.2D}, [x9], #32 + ld1 {V10.2D}, [x9] cmp w14, #1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_done beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1 cmp w14, #4 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w19, w19 - mov v15.s[3], w19 + mov V15.S[3], w19 rev w17, w17 - mov v16.s[3], w17 + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x0], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x0], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x0], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x0], #16 - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x0], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x0], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x0], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x0], #16 + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_4 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w19, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w17, w15, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w15, w15, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w20, w20 - rbit v19.16b, v19.16b - mov v14.s[3], w20 + rbit V19.16B, V19.16B + mov V14.S[3], w20 rev w19, w19 - rbit v20.16b, v20.16b - mov v15.s[3], w19 + rbit V20.16B, V20.16B + mov V15.S[3], w19 rev w17, w17 - rbit v21.16b, v21.16b - mov v16.s[3], w17 + rbit V21.16B, V21.16B + mov V16.S[3], w17 rev w16, w15 - mov v17.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w14, w14, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x0], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x0], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w14, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x1], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x1], #0x40 bge L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_4 L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w14, #1 beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_2 add w20, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w15, w15, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w20, w20 - mov v14.s[3], w20 + mov V14.S[3], w20 rev w16, w15 - mov v15.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w14, w14, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x0], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x0], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x1], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x0], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x0], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x1], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1 add w15, w15, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x0], #16 - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x1], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x0], #16 + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x1], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_128_done ands w14, w2, #15 beq L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B mov w19, w14 - st1 {v15.2d}, [x11] + st1 {V15.2D}, [x11] cmp x19, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_dw ldr x17, [x0], #8 @@ -24001,51 +24001,51 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_bytes sub x11, x11, x14 - ld1 {v15.2d}, [x11] + ld1 {V15.2D}, [x11] add w15, w15, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w16, w15 - mov v14.s[3], w16 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w16 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.2d}, [x11] + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.2D}, [x11] cmp w14, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_dw ldr x17, [x11], #8 @@ -24071,58 +24071,58 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done - ld1 {v14.2d}, [x12] + ld1 {V14.2D}, [x12] ubfiz x8, x8, #3, #32 rbit x8, x8 - mov v28.d[0], x8 + mov V28.D[0], x8 ubfiz x2, x2, #3, #32 rbit x2, x2 - mov v28.d[1], x2 - eor v26.16b, v26.16b, v28.16b - pmull v28.1q, v26.1d, v22.1d - pmull2 v29.1q, v26.2d, v22.2d - ext v31.16b, v26.16b, v26.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V28.D[1], x2 + eor V26.16B, V26.16B, V28.16B + pmull V28.1Q, V26.1D, V22.1D + pmull2 V29.1Q, V26.2D, V22.2D + ext8 V31.16B, V26.16B, V26.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - rbit v26.16b, v26.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v26.16b, v26.16b, v14.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + rbit V26.16B, V26.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V26.16B, V26.16B, V14.16B cmp w6, #16 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag - ld1 {v28.16b}, [x5] + ld1 {V28.16B}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag ubfiz x6, x6, #0, #32 - eor v28.16b, v28.16b, v28.16b + eor V28.16B, V28.16B, V28.16B mov x17, x6 - st1 {v28.2d}, [x11] + st1 {V28.2D}, [x11] cmp x17, #8 blt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_dw ldr x16, [x5], #8 @@ -24148,9 +24148,9 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_end_bytes sub x11, x11, x6 - ld1 {v28.2d}, [x11] + ld1 {V28.2D}, [x11] mov x17, #16 - st1 {v26.2d}, [x11] + st1 {V26.2D}, [x11] sub x17, x17, x6 add x11, x11, x6 L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte @@ -24158,11 +24158,11 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte subs x17, x17, #1 bne L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte subs x11, x11, #16 - ld1 {v26.2d}, [x11] + ld1 {V26.2D}, [x11] L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded - eor v28.16b, v28.16b, v26.16b - mov x16, v28.d[0] - mov x17, v28.d[1] + eor V28.16B, V28.16B, V26.16B + mov x16, V28.D[0] + mov x17, V28.D[1] mov w19, #-180 orr x16, x16, x17 cmp x16, #0 @@ -24174,10 +24174,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_done ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -24190,53 +24190,53 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_done AES_GCM_init_AARCH64 PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - movi v6.16b, #0x87 - ld1 {v5.2d}, [x4] - ushr v6.2d, v6.2d, #56 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + movi V6.16B, #0x87 + ld1 {V5.2D}, [x4] + ushr V6.2D, V6.2D, #56 ; Load Nonce cmp w3, #12 bne L_aes_gcm_init_arm64_crypto_ghash_nonce ldr x9, [x2] - movi v4.4s, #1, lsl 24 + movi V4.4S, #1, lsl 24 ldr w10, [x2, #8] - mov v4.d[0], x9 - mov v4.s[2], w10 + mov V4.D[0], x9 + mov V4.S[2], w10 mov w8, #1 b L_aes_gcm_init_arm64_crypto_done_nonce L_aes_gcm_init_arm64_crypto_ghash_nonce - eor v4.16b, v4.16b, v4.16b + eor V4.16B, V4.16B, V4.16B lsr w7, w3, #4 cbz w7, L_aes_gcm_init_arm64_crypto_done L_aes_gcm_init_arm64_crypto_start_1 - ld1 {v0.16b}, [x2], #16 - rbit v0.16b, v0.16b - eor v3.16b, v4.16b, v0.16b + ld1 {V0.16B}, [x2], #16 + rbit V0.16B, V0.16B + eor V3.16B, V4.16B, V0.16B ; X = C * H^1 - pmull v7.1q, v3.1d, v5.1d - pmull2 v8.1q, v3.2d, v5.2d - ext v10.16b, v3.16b, v3.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b + pmull V7.1Q, V3.1D, V5.1D + pmull2 V8.1Q, V3.2D, V5.2D + ext8 V10.16B, V3.16B, V3.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B ; Reduce - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor v10.16b, v10.16b, v8.16b - eor v10.16b, v10.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor V10.16B, V10.16B, V8.16B + eor V10.16B, V10.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B ; Done GHASH subs w7, w7, #1 bne L_aes_gcm_init_arm64_crypto_start_1 L_aes_gcm_init_arm64_crypto_done and w13, w3, #15 cbz x13, L_aes_gcm_init_arm64_crypto_partial_done - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B mov w12, w13 - st1 {v7.2d}, [x6] + st1 {V7.2D}, [x6] cmp w12, #8 blt L_aes_gcm_init_arm64_crypto_start_dw ldr x11, [x2], #8 @@ -24262,93 +24262,93 @@ L_aes_gcm_init_arm64_crypto_start_byte bne L_aes_gcm_init_arm64_crypto_start_byte L_aes_gcm_init_arm64_crypto_end_bytes sub x6, x6, x13 - ld1 {v0.2d}, [x6] - rbit v0.16b, v0.16b - eor v3.16b, v4.16b, v0.16b + ld1 {V0.2D}, [x6] + rbit V0.16B, V0.16B + eor V3.16B, V4.16B, V0.16B ; X = C * H^1 - pmull v7.1q, v3.1d, v5.1d - pmull2 v8.1q, v3.2d, v5.2d - ext v10.16b, v3.16b, v3.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b + pmull V7.1Q, V3.1D, V5.1D + pmull2 V8.1Q, V3.2D, V5.2D + ext8 V10.16B, V3.16B, V3.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B ; Reduce - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor v10.16b, v10.16b, v8.16b - eor v10.16b, v10.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor V10.16B, V10.16B, V8.16B + eor V10.16B, V10.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B ; Done GHASH L_aes_gcm_init_arm64_crypto_partial_done eor x7, x7, x7 ubfiz x13, x3, #3, #32 - mov v7.d[0], x7 - mov v7.d[1], x13 - rev64 v7.16b, v7.16b - rbit v7.16b, v7.16b - eor v4.16b, v4.16b, v7.16b - pmull v7.1q, v4.1d, v5.1d - pmull2 v8.1q, v4.2d, v5.2d - ext v10.16b, v4.16b, v4.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor v10.16b, v10.16b, v8.16b - eor v10.16b, v10.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b - rbit v4.16b, v4.16b - mov w8, v4.s[3] + mov V7.D[0], x7 + mov V7.D[1], x13 + rev64 V7.16B, V7.16B + rbit V7.16B, V7.16B + eor V4.16B, V4.16B, V7.16B + pmull V7.1Q, V4.1D, V5.1D + pmull2 V8.1Q, V4.2D, V5.2D + ext8 V10.16B, V4.16B, V4.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor V10.16B, V10.16B, V8.16B + eor V10.16B, V10.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B + rbit V4.16B, V4.16B + mov w8, V4.S[3] rev w8, w8 L_aes_gcm_init_arm64_crypto_done_nonce - st1 {v4.2d}, [x5] - ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b - aesmc v4.16b, v4.16b - aese v4.16b, v9.16b - aesmc v4.16b, v4.16b - aese v4.16b, v10.16b - aesmc v4.16b, v4.16b - ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b - aesmc v4.16b, v4.16b - aese v4.16b, v9.16b - aesmc v4.16b, v4.16b - aese v4.16b, v10.16b - aesmc v4.16b, v4.16b + st1 {V4.2D}, [x5] + ld1 {V7.2D, V8.2D, V9.2D, V10.2D}, [x0], #0x40 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B + aesmc V4.16B, V4.16B + aese V4.16B, V9.16B + aesmc V4.16B, V4.16B + aese V4.16B, V10.16B + aesmc V4.16B, V4.16B + ld1 {V7.2D, V8.2D, V9.2D, V10.2D}, [x0], #0x40 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B + aesmc V4.16B, V4.16B + aese V4.16B, V9.16B + aesmc V4.16B, V4.16B + aese V4.16B, V10.16B + aesmc V4.16B, V4.16B subs w1, w1, #10 - ld1 {v7.2d, v8.2d}, [x0], #32 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + ld1 {V7.2D, V8.2D}, [x0], #32 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B beq L_aes_gcm_init_arm64_crypto_round_done - ld1 {v7.2d, v8.2d}, [x0], #32 + ld1 {V7.2D, V8.2D}, [x0], #32 subs w1, w1, #2 - aesmc v4.16b, v4.16b - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + aesmc V4.16B, V4.16B + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B beq L_aes_gcm_init_arm64_crypto_round_done - ld1 {v7.2d, v8.2d}, [x0], #32 - aesmc v4.16b, v4.16b - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + ld1 {V7.2D, V8.2D}, [x0], #32 + aesmc V4.16B, V4.16B + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B L_aes_gcm_init_arm64_crypto_round_done - ld1 {v7.2d}, [x0] - eor v4.16b, v4.16b, v7.16b - st1 {v4.2d}, [x6] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] + ld1 {V7.2D}, [x0] + eor V4.16B, V4.16B, V7.16B + st1 {V4.2D}, [x6] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] ldp x29, x30, [sp], #48 ret ENDP @@ -24358,32 +24358,32 @@ L_aes_gcm_init_arm64_crypto_round_done AES_GCM_ghash_block_AARCH64 PROC stp x29, x30, [sp, #-32]! add x29, sp, #0 - stp d8, d9, [x29, #16] - ld1 {v6.2d}, [x1] - movi v7.16b, #0x87 - ld1 {v5.2d}, [x2] - ushr v7.2d, v7.2d, #56 - ld1 {v4.2d}, [x0] - rbit v4.16b, v4.16b - eor v8.16b, v6.16b, v4.16b + stp D8, D9, [x29, #16] + ld1 {V6.2D}, [x1] + movi V7.16B, #0x87 + ld1 {V5.2D}, [x2] + ushr V7.2D, V7.2D, #56 + ld1 {V4.2D}, [x0] + rbit V4.16B, V4.16B + eor V8.16B, V6.16B, V4.16B ; X = C * H^1 - pmull v0.1q, v8.1d, v5.1d - pmull2 v1.1q, v8.2d, v5.2d - ext v3.16b, v8.16b, v8.16b, #8 - pmull v2.1q, v3.1d, v5.1d - pmull2 v3.1q, v3.2d, v5.2d - eor v2.16b, v2.16b, v3.16b + pmull V0.1Q, V8.1D, V5.1D + pmull2 V1.1Q, V8.2D, V5.2D + ext8 V3.16B, V8.16B, V8.16B, #8 + pmull V2.1Q, V3.1D, V5.1D + pmull2 V3.1Q, V3.2D, V5.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v7.2d - eor v3.16b, v3.16b, v1.16b - eor v3.16b, v3.16b, v2.16b - pmull2 v2.1q, v3.2d, v7.2d - mov v0.d[1], v3.d[0] - eor v6.16b, v0.16b, v2.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V7.2D + eor V3.16B, V3.16B, V1.16B + eor V3.16B, V3.16B, V2.16B + pmull2 V2.1Q, V3.2D, V7.2D + mov V0.D[1], V3.D[0] + eor V6.16B, V0.16B, V2.16B ; Done GHASH - st1 {v6.2d}, [x1] - ldp d8, d9, [x29, #16] + st1 {V6.2D}, [x1] + ldp D8, D9, [x29, #16] ldp x29, x30, [sp], #32 ret ENDP @@ -24393,102 +24393,102 @@ AES_GCM_ghash_block_AARCH64 PROC AES_GCM_aad_update_AARCH64 PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] - ld1 {v20.2d}, [x2] - movi v21.16b, #0x87 - ld1 {v12.2d}, [x3] - ushr v21.2d, v21.2d, #56 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + ld1 {V20.2D}, [x2] + movi V21.16B, #0x87 + ld1 {V12.2D}, [x3] + ushr V21.2D, V21.2D, #56 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_h_done ; Square H => H^2 - pmull2 v11.1q, v12.2d, v12.2d - pmull v10.1q, v12.1d, v12.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v13.16b, v10.16b, v11.16b + pmull2 V11.1Q, V12.2D, V12.2D + pmull V10.1Q, V12.1D, V12.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V13.16B, V10.16B, V11.16B cmp x1, #0x100 blt L_aes_gcm_aad_update_arm64_crypto_h_done ; Multiply H and H^2 => H^3 - pmull v8.1q, v12.1d, v13.1d - pmull2 v9.1q, v12.2d, v13.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v13.1d - pmull2 v11.1q, v11.2d, v13.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V13.1D + pmull2 V9.1Q, V12.2D, V13.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V13.1D + pmull2 V11.1Q, V11.2D, V13.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v14.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V14.16B, V8.16B, V10.16B ; Square H^2 => H^4 - pmull2 v11.1q, v13.2d, v13.2d - pmull v10.1q, v13.1d, v13.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v15.16b, v10.16b, v11.16b + pmull2 V11.1Q, V13.2D, V13.2D + pmull V10.1Q, V13.1D, V13.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V15.16B, V10.16B, V11.16B ; Done cmp x1, #0x400 blt L_aes_gcm_aad_update_arm64_crypto_h_done ; Multiply H and H^4 => H^5 - pmull v8.1q, v12.1d, v15.1d - pmull2 v9.1q, v12.2d, v15.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v15.1d - pmull2 v11.1q, v11.2d, v15.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V15.1D + pmull2 V9.1Q, V12.2D, V15.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V15.1D + pmull2 V11.1Q, V11.2D, V15.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v16.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V16.16B, V8.16B, V10.16B ; Square H^3 => H^6 - pmull2 v11.1q, v14.2d, v14.2d - pmull v10.1q, v14.1d, v14.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v17.16b, v10.16b, v11.16b + pmull2 V11.1Q, V14.2D, V14.2D + pmull V10.1Q, V14.1D, V14.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V17.16B, V10.16B, V11.16B ; Multiply H and H^6 => H^7 - pmull v8.1q, v12.1d, v17.1d - pmull2 v9.1q, v12.2d, v17.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v17.1d - pmull2 v11.1q, v11.2d, v17.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V17.1D + pmull2 V9.1Q, V12.2D, V17.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V17.1D + pmull2 V11.1Q, V11.2D, V17.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v18.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V18.16B, V8.16B, V10.16B ; Square H^4 => H^8 - pmull2 v11.1q, v15.2d, v15.2d - pmull v10.1q, v15.1d, v15.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v19.16b, v10.16b, v11.16b + pmull2 V11.1Q, V15.2D, V15.2D + pmull V10.1Q, V15.1D, V15.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V19.16B, V10.16B, V11.16B ; Done L_aes_gcm_aad_update_arm64_crypto_h_done lsr x1, x1, #4 @@ -24499,102 +24499,102 @@ L_aes_gcm_aad_update_arm64_crypto_h_done cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_start_4 L_aes_gcm_aad_update_arm64_crypto_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - rbit v4.16b, v4.16b - rbit v5.16b, v5.16b - rbit v6.16b, v6.16b - rbit v7.16b, v7.16b - eor v0.16b, v0.16b, v20.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + rbit V4.16B, V4.16B + rbit V5.16B, V5.16B + rbit V6.16B, V6.16B + rbit V7.16B, V7.16B + eor V0.16B, V0.16B, V20.16B ; X = C * H^1 - pmull v8.1q, v7.1d, v12.1d - pmull2 v9.1q, v7.2d, v12.2d - ext v11.16b, v7.16b, v7.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V7.1D, V12.1D + pmull2 V9.1Q, V7.2D, V12.2D + ext8 V11.16B, V7.16B, V7.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v6.1d - pmull2 v20.1q, v13.2d, v6.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v6.16b, v6.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V13.1D, V6.1D + pmull2 V20.1Q, V13.2D, V6.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V6.16B, V6.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^3 - pmull v11.1q, v14.1d, v5.1d - pmull2 v20.1q, v14.2d, v5.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v5.16b, v5.16b, #8 - pmull v11.1q, v20.1d, v14.1d - pmull2 v20.1q, v20.2d, v14.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V14.1D, V5.1D + pmull2 V20.1Q, V14.2D, V5.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V5.16B, V5.16B, #8 + pmull V11.1Q, V20.1D, V14.1D + pmull2 V20.1Q, V20.2D, V14.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^4 - pmull v11.1q, v15.1d, v4.1d - pmull2 v20.1q, v15.2d, v4.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v4.16b, v4.16b, #8 - pmull v11.1q, v20.1d, v15.1d - pmull2 v20.1q, v20.2d, v15.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V15.1D, V4.1D + pmull2 V20.1Q, V15.2D, V4.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V4.16B, V4.16B, #8 + pmull V11.1Q, V20.1D, V15.1D + pmull2 V20.1Q, V20.2D, V15.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^5 - pmull v11.1q, v16.1d, v3.1d - pmull2 v20.1q, v16.2d, v3.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v3.16b, v3.16b, #8 - pmull v11.1q, v20.1d, v16.1d - pmull2 v20.1q, v20.2d, v16.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V16.1D, V3.1D + pmull2 V20.1Q, V16.2D, V3.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V3.16B, V3.16B, #8 + pmull V11.1Q, V20.1D, V16.1D + pmull2 V20.1Q, V20.2D, V16.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^6 - pmull v11.1q, v17.1d, v2.1d - pmull2 v20.1q, v17.2d, v2.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v2.16b, v2.16b, #8 - pmull v11.1q, v20.1d, v17.1d - pmull2 v20.1q, v20.2d, v17.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V17.1D, V2.1D + pmull2 V20.1Q, V17.2D, V2.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V2.16B, V2.16B, #8 + pmull V11.1Q, V20.1D, V17.1D + pmull2 V20.1Q, V20.2D, V17.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^7 - pmull v11.1q, v18.1d, v1.1d - pmull2 v20.1q, v18.2d, v1.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v1.16b, v1.16b, #8 - pmull v11.1q, v20.1d, v18.1d - pmull2 v20.1q, v20.2d, v18.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V18.1D, V1.1D + pmull2 V20.1Q, V18.2D, V1.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V1.16B, V1.16B, #8 + pmull V11.1Q, V20.1D, V18.1D + pmull2 V20.1Q, V20.2D, V18.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^8 - pmull v11.1q, v19.1d, v0.1d - pmull2 v20.1q, v19.2d, v0.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v0.16b, v0.16b, #8 - pmull v11.1q, v20.1d, v19.1d - pmull2 v20.1q, v20.2d, v19.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V19.1D, V0.1D + pmull2 V20.1Q, V19.2D, V0.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V0.16B, V0.16B, #8 + pmull V11.1Q, V20.1D, V19.1D + pmull2 V20.1Q, V20.2D, V19.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #8 cmp x1, #8 @@ -24605,57 +24605,57 @@ L_aes_gcm_aad_update_arm64_crypto_start_8 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_start_2 L_aes_gcm_aad_update_arm64_crypto_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v0.16b, v0.16b, v20.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V0.16B, V0.16B, V20.16B ; X = C * H^1 - pmull v8.1q, v3.1d, v12.1d - pmull2 v9.1q, v3.2d, v12.2d - ext v11.16b, v3.16b, v3.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V3.1D, V12.1D + pmull2 V9.1Q, V3.2D, V12.2D + ext8 V11.16B, V3.16B, V3.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v2.1d - pmull2 v20.1q, v13.2d, v2.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v2.16b, v2.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V13.1D, V2.1D + pmull2 V20.1Q, V13.2D, V2.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V2.16B, V2.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^3 - pmull v11.1q, v14.1d, v1.1d - pmull2 v20.1q, v14.2d, v1.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v1.16b, v1.16b, #8 - pmull v11.1q, v20.1d, v14.1d - pmull2 v20.1q, v20.2d, v14.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V14.1D, V1.1D + pmull2 V20.1Q, V14.2D, V1.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V1.16B, V1.16B, #8 + pmull V11.1Q, V20.1D, V14.1D + pmull2 V20.1Q, V20.2D, V14.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; X += C * H^4 - pmull v11.1q, v15.1d, v0.1d - pmull2 v20.1q, v15.2d, v0.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v0.16b, v0.16b, #8 - pmull v11.1q, v20.1d, v15.1d - pmull2 v20.1q, v20.2d, v15.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V15.1D, V0.1D + pmull2 V20.1Q, V15.2D, V0.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V0.16B, V0.16B, #8 + pmull V11.1Q, V20.1D, V15.1D + pmull2 V20.1Q, V20.2D, V15.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #4 cmp x1, #4 @@ -24664,35 +24664,35 @@ L_aes_gcm_aad_update_arm64_crypto_start_4 blt L_aes_gcm_aad_update_arm64_crypto_done beq L_aes_gcm_aad_update_arm64_crypto_start_1 L_aes_gcm_aad_update_arm64_crypto_start_2 - ld1 {v0.16b, v1.16b}, [x0], #32 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - eor v3.16b, v20.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + eor V3.16B, V20.16B, V0.16B ; X = C * H^1 - pmull v8.1q, v1.1d, v12.1d - pmull2 v9.1q, v1.2d, v12.2d - ext v11.16b, v1.16b, v1.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V1.1D, V12.1D + pmull2 V9.1Q, V1.2D, V12.2D + ext8 V11.16B, V1.16B, V1.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v3.1d - pmull2 v20.1q, v13.2d, v3.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v3.16b, v3.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor v20.16b, v20.16b, v11.16b - eor v10.16b, v10.16b, v20.16b + pmull V11.1Q, V13.1D, V3.1D + pmull2 V20.1Q, V13.2D, V3.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V3.16B, V3.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor V20.16B, V20.16B, V11.16B + eor V10.16B, V10.16B, V20.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #2 cmp x1, #1 @@ -24701,33 +24701,33 @@ L_aes_gcm_aad_update_arm64_crypto_start_2 L_aes_gcm_aad_update_arm64_crypto_start_1 cbz x1, L_aes_gcm_aad_update_arm64_crypto_done L_aes_gcm_aad_update_arm64_crypto_both_1 - ld1 {v0.16b}, [x0], #16 - rbit v0.16b, v0.16b - eor v3.16b, v20.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + rbit V0.16B, V0.16B + eor V3.16B, V20.16B, V0.16B ; X = C * H^1 - pmull v8.1q, v3.1d, v12.1d - pmull2 v9.1q, v3.2d, v12.2d - ext v11.16b, v3.16b, v3.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V3.1D, V12.1D + pmull2 V9.1Q, V3.2D, V12.2D + ext8 V11.16B, V3.16B, V3.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor v11.16b, v11.16b, v9.16b - eor v11.16b, v11.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor V11.16B, V11.16B, V9.16B + eor V11.16B, V11.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH subs x1, x1, #1 bne L_aes_gcm_aad_update_arm64_crypto_both_1 L_aes_gcm_aad_update_arm64_crypto_done - st1 {v20.2d}, [x2] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st1 {V20.2D}, [x2] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -24735,55 +24735,55 @@ L_aes_gcm_aad_update_arm64_crypto_done ALIGN 4 EXPORT AES_GCM_encrypt_block_AARCH64 AES_GCM_encrypt_block_AARCH64 PROC - ld1 {v5.2d}, [x4] - ld1 {v4.2d}, [x3] - mov w5, v5.s[3] + ld1 {V5.2D}, [x4] + ld1 {V4.2D}, [x3] + mov w5, V5.S[3] rev w5, w5 add w5, w5, #1 rev w5, w5 - mov v5.s[3], w5 - st1 {v5.2d}, [x4] - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b - aesmc v5.16b, v5.16b - aese v5.16b, v2.16b - aesmc v5.16b, v5.16b - aese v5.16b, v3.16b - aesmc v5.16b, v5.16b - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b - aesmc v5.16b, v5.16b - aese v5.16b, v2.16b - aesmc v5.16b, v5.16b - aese v5.16b, v3.16b - aesmc v5.16b, v5.16b + mov V5.S[3], w5 + st1 {V5.2D}, [x4] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B + aesmc V5.16B, V5.16B + aese V5.16B, V2.16B + aesmc V5.16B, V5.16B + aese V5.16B, V3.16B + aesmc V5.16B, V5.16B + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B + aesmc V5.16B, V5.16B + aese V5.16B, V2.16B + aesmc V5.16B, V5.16B + aese V5.16B, V3.16B + aesmc V5.16B, V5.16B subs w1, w1, #10 - ld1 {v0.2d, v1.2d}, [x0], #32 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + ld1 {V0.2D, V1.2D}, [x0], #32 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B beq L_aes_gcm_encrypt_block_arm64_crypto_round_done - ld1 {v0.2d, v1.2d}, [x0], #32 + ld1 {V0.2D, V1.2D}, [x0], #32 subs w1, w1, #2 - aesmc v5.16b, v5.16b - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + aesmc V5.16B, V5.16B + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B beq L_aes_gcm_encrypt_block_arm64_crypto_round_done - ld1 {v0.2d, v1.2d}, [x0], #32 - aesmc v5.16b, v5.16b - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + ld1 {V0.2D, V1.2D}, [x0], #32 + aesmc V5.16B, V5.16B + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B L_aes_gcm_encrypt_block_arm64_crypto_round_done - ld1 {v0.2d}, [x0] - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v5.16b - st1 {v4.2d}, [x2] + ld1 {V0.2D}, [x0] + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V5.16B + st1 {V4.2D}, [x2] ret ENDP AREA |.text|, CODE, READONLY @@ -24793,105 +24793,105 @@ AES_GCM_encrypt_update_AARCH64 PROC stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] - stp d8, d9, [x29, #32] - stp d10, d11, [x29, #48] - stp d12, d13, [x29, #64] - stp d14, d15, [x29, #80] - ld1 {v13.2d}, [x7] - movi v27.16b, #0x87 - ld1 {v26.2d}, [x5] - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x6] - mov w9, v13.s[3] + stp D8, D9, [x29, #32] + stp D10, D11, [x29, #48] + stp D12, D13, [x29, #64] + stp D14, D15, [x29, #80] + ld1 {V13.2D}, [x7] + movi V27.16B, #0x87 + ld1 {V26.2D}, [x5] + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x6] + mov w9, V13.S[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w4, #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w4, #0x200 blt L_aes_gcm_encrypt_update_arm64_crypto_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_encrypt_update_arm64_crypto_h_done lsr w8, w4, #4 @@ -24903,23 +24903,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_h_done cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_192_start_4 L_aes_gcm_encrypt_update_arm64_crypto_192_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -24928,1195 +24928,1195 @@ L_aes_gcm_encrypt_update_arm64_crypto_192_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_192_end_8 L_aes_gcm_encrypt_update_arm64_crypto_192_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_192_both_8 L_aes_gcm_encrypt_update_arm64_crypto_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_192_done beq L_aes_gcm_encrypt_update_arm64_crypto_192_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_192_end_4 L_aes_gcm_encrypt_update_arm64_crypto_192_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_192_both_4 L_aes_gcm_encrypt_update_arm64_crypto_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_192_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_192_done L_aes_gcm_encrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_192_done L_aes_gcm_encrypt_update_arm64_crypto_192_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_192_done ENDIF @@ -26127,23 +26127,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_start_256 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_256_start_4 L_aes_gcm_encrypt_update_arm64_crypto_256_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -26152,1312 +26152,1312 @@ L_aes_gcm_encrypt_update_arm64_crypto_256_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_256_end_8 L_aes_gcm_encrypt_update_arm64_crypto_256_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_256_both_8 L_aes_gcm_encrypt_update_arm64_crypto_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0], #16 cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_256_done beq L_aes_gcm_encrypt_update_arm64_crypto_256_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_256_end_4 L_aes_gcm_encrypt_update_arm64_crypto_256_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_256_both_4 L_aes_gcm_encrypt_update_arm64_crypto_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_256_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_256_done L_aes_gcm_encrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_256_done L_aes_gcm_encrypt_update_arm64_crypto_256_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x0, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x0, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_256_done ENDIF @@ -27468,23 +27468,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_start_128 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_128_start_4 L_aes_gcm_encrypt_update_arm64_crypto_128_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -27493,1096 +27493,1096 @@ L_aes_gcm_encrypt_update_arm64_crypto_128_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_128_end_8 L_aes_gcm_encrypt_update_arm64_crypto_128_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v30.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - mov v28.d[1], v31.d[0] - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V30.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + mov V28.D[1], V31.D[0] + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_128_both_8 L_aes_gcm_encrypt_update_arm64_crypto_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d}, [x0], #32 - ld1 {v10.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D}, [x0], #32 + ld1 {V10.2D}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_128_done beq L_aes_gcm_encrypt_update_arm64_crypto_128_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_128_end_4 L_aes_gcm_encrypt_update_arm64_crypto_128_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_128_both_4 L_aes_gcm_encrypt_update_arm64_crypto_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_128_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_128_done L_aes_gcm_encrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_128_done L_aes_gcm_encrypt_update_arm64_crypto_128_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x3], #16 - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x3], #16 + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_128_done ENDIF L_aes_gcm_encrypt_update_arm64_crypto_done rev w9, w9 - mov v13.s[3], w9 - st1 {v26.2d}, [x5] - st1 {v13.2d}, [x7] + mov V13.S[3], w9 + st1 {V26.2D}, [x5] + st1 {V13.2D}, [x7] ldr x17, [x29, #24] - ldp d8, d9, [x29, #32] - ldp d10, d11, [x29, #48] - ldp d12, d13, [x29, #64] - ldp d14, d15, [x29, #80] + ldp D8, D9, [x29, #32] + ldp D10, D11, [x29, #48] + ldp D12, D13, [x29, #64] + ldp D14, D15, [x29, #80] ldp x29, x30, [sp], #0x60 ret ENDP @@ -28590,40 +28590,40 @@ L_aes_gcm_encrypt_update_arm64_crypto_done ALIGN 4 EXPORT AES_GCM_encrypt_final_AARCH64 AES_GCM_encrypt_final_AARCH64 PROC - ld1 {v5.2d}, [x0] - movi v6.16b, #0x87 - ld1 {v4.2d}, [x5] - ushr v6.2d, v6.2d, #56 - ld1 {v7.2d}, [x6] + ld1 {V5.2D}, [x0] + movi V6.16B, #0x87 + ld1 {V4.2D}, [x5] + ushr V6.2D, V6.2D, #56 + ld1 {V7.2D}, [x6] ubfiz x4, x4, #3, #32 rbit x4, x4 - mov v0.d[0], x4 + mov V0.D[0], x4 ubfiz x3, x3, #3, #32 rbit x3, x3 - mov v0.d[1], x3 - eor v5.16b, v5.16b, v0.16b - pmull v0.1q, v5.1d, v4.1d - pmull2 v1.1q, v5.2d, v4.2d - ext v3.16b, v5.16b, v5.16b, #8 - pmull v2.1q, v3.1d, v4.1d - pmull2 v3.1q, v3.2d, v4.2d - eor v2.16b, v2.16b, v3.16b + mov V0.D[1], x3 + eor V5.16B, V5.16B, V0.16B + pmull V0.1Q, V5.1D, V4.1D + pmull2 V1.1Q, V5.2D, V4.2D + ext8 V3.16B, V5.16B, V5.16B, #8 + pmull V2.1Q, V3.1D, V4.1D + pmull2 V3.1Q, V3.2D, V4.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v6.2d - eor v3.16b, v3.16b, v1.16b - eor v3.16b, v3.16b, v2.16b - pmull2 v2.1q, v3.2d, v6.2d - mov v0.d[1], v3.d[0] - eor v5.16b, v0.16b, v2.16b - rbit v5.16b, v5.16b - eor v5.16b, v5.16b, v7.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V6.2D + eor V3.16B, V3.16B, V1.16B + eor V3.16B, V3.16B, V2.16B + pmull2 V2.1Q, V3.2D, V6.2D + mov V0.D[1], V3.D[0] + eor V5.16B, V0.16B, V2.16B + rbit V5.16B, V5.16B + eor V5.16B, V5.16B, V7.16B cmp w2, #16 bne L_aes_gcm_encrypt_final_arm64_crypto_tag_partial - st1 {v5.16b}, [x1] + st1 {V5.16B}, [x1] b L_aes_gcm_encrypt_final_arm64_crypto_done L_aes_gcm_encrypt_final_arm64_crypto_tag_partial - st1 {v5.16b}, [x0] + st1 {V5.16B}, [x0] cmp w2, #8 blt L_aes_gcm_encrypt_final_arm64_crypto_tag_start_dw ldr x8, [x0], #8 @@ -28658,105 +28658,105 @@ AES_GCM_decrypt_update_AARCH64 PROC stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] - stp d8, d9, [x29, #32] - stp d10, d11, [x29, #48] - stp d12, d13, [x29, #64] - stp d14, d15, [x29, #80] - ld1 {v13.2d}, [x7] - movi v27.16b, #0x87 - ld1 {v26.2d}, [x5] - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x6] - mov w9, v13.s[3] + stp D8, D9, [x29, #32] + stp D10, D11, [x29, #48] + stp D12, D13, [x29, #64] + stp D14, D15, [x29, #80] + ld1 {V13.2D}, [x7] + movi V27.16B, #0x87 + ld1 {V26.2D}, [x5] + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x6] + mov w9, V13.S[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w4, #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w4, #0x200 blt L_aes_gcm_decrypt_update_arm64_crypto_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_decrypt_update_arm64_crypto_h_done lsr w8, w4, #4 @@ -28768,1222 +28768,1222 @@ L_aes_gcm_decrypt_update_arm64_crypto_h_done cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_192_start_4 L_aes_gcm_decrypt_update_arm64_crypto_192_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_192_end_8 L_aes_gcm_decrypt_update_arm64_crypto_192_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_192_both_8 L_aes_gcm_decrypt_update_arm64_crypto_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_192_done beq L_aes_gcm_decrypt_update_arm64_crypto_192_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_192_end_4 L_aes_gcm_decrypt_update_arm64_crypto_192_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_192_both_4 L_aes_gcm_decrypt_update_arm64_crypto_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_192_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_192_done L_aes_gcm_decrypt_update_arm64_crypto_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_192_done L_aes_gcm_decrypt_update_arm64_crypto_192_start_1 - ld1 {v15.16b}, [x3], #16 + ld1 {V15.16B}, [x3], #16 add w9, w9, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v29.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V29.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.16b}, [x2], #16 + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.16B}, [x2], #16 L_aes_gcm_decrypt_update_arm64_crypto_192_done ENDIF b L_aes_gcm_decrypt_update_arm64_crypto_done @@ -29993,1337 +29993,1337 @@ L_aes_gcm_decrypt_update_arm64_crypto_start_256 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_256_start_4 L_aes_gcm_decrypt_update_arm64_crypto_256_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_256_end_8 L_aes_gcm_decrypt_update_arm64_crypto_256_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_256_both_8 L_aes_gcm_decrypt_update_arm64_crypto_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0], #16 cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_256_done beq L_aes_gcm_decrypt_update_arm64_crypto_256_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_256_end_4 L_aes_gcm_decrypt_update_arm64_crypto_256_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_256_both_4 L_aes_gcm_decrypt_update_arm64_crypto_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_256_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_256_done L_aes_gcm_decrypt_update_arm64_crypto_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_256_done L_aes_gcm_decrypt_update_arm64_crypto_256_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x0, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x0, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_256_done ENDIF @@ -31334,1121 +31334,1121 @@ L_aes_gcm_decrypt_update_arm64_crypto_start_128 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_128_start_4 L_aes_gcm_decrypt_update_arm64_crypto_128_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_128_end_8 L_aes_gcm_decrypt_update_arm64_crypto_128_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v26.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v30.16b, v30.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V26.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V30.16B, V30.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v26.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V26.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v26.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v30.16b, v30.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V26.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V30.16B, V30.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v26.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v30.16b, v30.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V26.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V30.16B, V30.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v26.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V26.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v31.16b, v31.16b, v29.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V31.16B, V31.16B, V29.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v31.16b, v31.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V31.16B, V31.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_128_both_8 L_aes_gcm_decrypt_update_arm64_crypto_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d}, [x0], #32 - ld1 {v10.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D}, [x0], #32 + ld1 {V10.2D}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_128_done beq L_aes_gcm_decrypt_update_arm64_crypto_128_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_128_end_4 L_aes_gcm_decrypt_update_arm64_crypto_128_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor v26.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v26.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor V26.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V26.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor v26.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v26.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor V26.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V26.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor v31.16b, v31.16b, v29.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - eor v31.16b, v31.16b, v30.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor V31.16B, V31.16B, V29.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + eor V31.16B, V31.16B, V30.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - eor v26.16b, v28.16b, v30.16b + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + eor V26.16B, V28.16B, V30.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_128_both_4 L_aes_gcm_decrypt_update_arm64_crypto_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_128_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_128_done L_aes_gcm_decrypt_update_arm64_crypto_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor v26.16b, v26.16b, v31.16b - eor v30.16b, v30.16b, v26.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor V26.16B, V26.16B, V31.16B + eor V30.16B, V30.16B, V26.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_128_done L_aes_gcm_decrypt_update_arm64_crypto_128_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x3], #16 - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x3], #16 + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor v31.16b, v31.16b, v29.16b - eor v31.16b, v31.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor V31.16B, V31.16B, V29.16B + eor V31.16B, V31.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_128_done ENDIF L_aes_gcm_decrypt_update_arm64_crypto_done rev w9, w9 - mov v13.s[3], w9 - st1 {v26.2d}, [x5] - st1 {v13.2d}, [x7] + mov V13.S[3], w9 + st1 {V26.2D}, [x5] + st1 {V13.2D}, [x7] ldr x17, [x29, #24] - ldp d8, d9, [x29, #32] - ldp d10, d11, [x29, #48] - ldp d12, d13, [x29, #64] - ldp d14, d15, [x29, #80] + ldp D8, D9, [x29, #32] + ldp D10, D11, [x29, #48] + ldp D12, D13, [x29, #64] + ldp D14, D15, [x29, #80] ldp x29, x30, [sp], #0x60 ret ENDP @@ -32456,43 +32456,43 @@ L_aes_gcm_decrypt_update_arm64_crypto_done ALIGN 4 EXPORT AES_GCM_decrypt_final_AARCH64 AES_GCM_decrypt_final_AARCH64 PROC - ld1 {v5.2d}, [x0] - movi v6.16b, #0x87 - ld1 {v4.2d}, [x5] - ushr v6.2d, v6.2d, #56 - ld1 {v7.2d}, [x6] + ld1 {V5.2D}, [x0] + movi V6.16B, #0x87 + ld1 {V4.2D}, [x5] + ushr V6.2D, V6.2D, #56 + ld1 {V7.2D}, [x6] ubfiz x4, x4, #3, #32 rbit x4, x4 - mov v0.d[0], x4 + mov V0.D[0], x4 ubfiz x3, x3, #3, #32 rbit x3, x3 - mov v0.d[1], x3 - eor v5.16b, v5.16b, v0.16b - pmull v0.1q, v5.1d, v4.1d - pmull2 v1.1q, v5.2d, v4.2d - ext v3.16b, v5.16b, v5.16b, #8 - pmull v2.1q, v3.1d, v4.1d - pmull2 v3.1q, v3.2d, v4.2d - eor v2.16b, v2.16b, v3.16b + mov V0.D[1], x3 + eor V5.16B, V5.16B, V0.16B + pmull V0.1Q, V5.1D, V4.1D + pmull2 V1.1Q, V5.2D, V4.2D + ext8 V3.16B, V5.16B, V5.16B, #8 + pmull V2.1Q, V3.1D, V4.1D + pmull2 V3.1Q, V3.2D, V4.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v6.2d - eor v3.16b, v3.16b, v1.16b - eor v3.16b, v3.16b, v2.16b - pmull2 v2.1q, v3.2d, v6.2d - mov v0.d[1], v3.d[0] - eor v5.16b, v0.16b, v2.16b - rbit v5.16b, v5.16b - eor v5.16b, v5.16b, v7.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V6.2D + eor V3.16B, V3.16B, V1.16B + eor V3.16B, V3.16B, V2.16B + pmull2 V2.1Q, V3.2D, V6.2D + mov V0.D[1], V3.D[0] + eor V5.16B, V0.16B, V2.16B + rbit V5.16B, V5.16B + eor V5.16B, V5.16B, V7.16B cmp w2, #16 blt L_aes_gcm_decrypt_final_arm64_crypto_part_tag - ld1 {v0.16b}, [x1] + ld1 {V0.16B}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_part_tag ubfiz x2, x2, #0, #32 - eor v0.16b, v0.16b, v0.16b + eor V0.16B, V0.16B, V0.16B mov x10, x2 - st1 {v0.2d}, [x0] + st1 {V0.2D}, [x0] cmp x10, #8 blt L_aes_gcm_decrypt_final_arm64_crypto_tag_start_dw ldr x9, [x1], #8 @@ -32518,9 +32518,9 @@ L_aes_gcm_decrypt_final_arm64_crypto_tag_start_byte bne L_aes_gcm_decrypt_final_arm64_crypto_tag_start_byte L_aes_gcm_decrypt_final_arm64_crypto_tag_end_bytes sub x0, x0, x2 - ld1 {v0.2d}, [x0] + ld1 {V0.2D}, [x0] mov x10, #16 - st1 {v5.2d}, [x0] + st1 {V5.2D}, [x0] sub x10, x10, x2 add x0, x0, x2 L_aes_gcm_decrypt_final_arm64_crypto_calc_tag_byte @@ -32528,11 +32528,11 @@ L_aes_gcm_decrypt_final_arm64_crypto_calc_tag_byte subs x10, x10, #1 bne L_aes_gcm_decrypt_final_arm64_crypto_calc_tag_byte subs x0, x0, #16 - ld1 {v5.2d}, [x0] + ld1 {V5.2D}, [x0] L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded - eor v0.16b, v0.16b, v5.16b - mov x9, v0.d[0] - mov x10, v0.d[1] + eor V0.16B, V0.16B, V5.16B + mov x9, V0.D[0] + mov x10, V0.D[1] mov w11, #-180 orr x9, x9, x10 cmp x9, #0 @@ -32549,52 +32549,52 @@ L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded AES_GCM_init_AARCH64_EOR3 PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - movi v6.16b, #0x87 - ld1 {v5.2d}, [x4] - ushr v6.2d, v6.2d, #56 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + movi V6.16B, #0x87 + ld1 {V5.2D}, [x4] + ushr V6.2D, V6.2D, #56 ; Load Nonce cmp w3, #12 bne L_aes_gcm_init_arm64_crypto_eor3_ghash_nonce ldr x9, [x2] - movi v4.4s, #1, lsl 24 + movi V4.4S, #1, lsl 24 ldr w10, [x2, #8] - mov v4.d[0], x9 - mov v4.s[2], w10 + mov V4.D[0], x9 + mov V4.S[2], w10 mov w8, #1 b L_aes_gcm_init_arm64_crypto_eor3_done_nonce L_aes_gcm_init_arm64_crypto_eor3_ghash_nonce - eor v4.16b, v4.16b, v4.16b + eor V4.16B, V4.16B, V4.16B lsr w7, w3, #4 cbz w7, L_aes_gcm_init_arm64_crypto_eor3_done L_aes_gcm_init_arm64_crypto_eor3_start_1 - ld1 {v0.16b}, [x2], #16 - rbit v0.16b, v0.16b - eor v3.16b, v4.16b, v0.16b + ld1 {V0.16B}, [x2], #16 + rbit V0.16B, V0.16B + eor V3.16B, V4.16B, V0.16B ; X = C * H^1 - pmull v7.1q, v3.1d, v5.1d - pmull2 v8.1q, v3.2d, v5.2d - ext v10.16b, v3.16b, v3.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b + pmull V7.1Q, V3.1D, V5.1D + pmull2 V8.1Q, V3.2D, V5.2D + ext8 V10.16B, V3.16B, V3.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B ; Reduce - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor3 v10.16b, v10.16b, v8.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor3 V10.16B, V10.16B, V8.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B ; Done GHASH subs w7, w7, #1 bne L_aes_gcm_init_arm64_crypto_eor3_start_1 L_aes_gcm_init_arm64_crypto_eor3_done and w13, w3, #15 cbz x13, L_aes_gcm_init_arm64_crypto_eor3_partial_done - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B mov w12, w13 - st1 {v7.2d}, [x6] + st1 {V7.2D}, [x6] cmp w12, #8 blt L_aes_gcm_init_arm64_crypto_eor3_start_dw ldr x11, [x2], #8 @@ -32620,91 +32620,91 @@ L_aes_gcm_init_arm64_crypto_eor3_start_byte bne L_aes_gcm_init_arm64_crypto_eor3_start_byte L_aes_gcm_init_arm64_crypto_eor3_end_bytes sub x6, x6, x13 - ld1 {v0.2d}, [x6] - rbit v0.16b, v0.16b - eor v3.16b, v4.16b, v0.16b + ld1 {V0.2D}, [x6] + rbit V0.16B, V0.16B + eor V3.16B, V4.16B, V0.16B ; X = C * H^1 - pmull v7.1q, v3.1d, v5.1d - pmull2 v8.1q, v3.2d, v5.2d - ext v10.16b, v3.16b, v3.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b + pmull V7.1Q, V3.1D, V5.1D + pmull2 V8.1Q, V3.2D, V5.2D + ext8 V10.16B, V3.16B, V3.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B ; Reduce - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor3 v10.16b, v10.16b, v8.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor3 V10.16B, V10.16B, V8.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B ; Done GHASH L_aes_gcm_init_arm64_crypto_eor3_partial_done eor x7, x7, x7 ubfiz x13, x3, #3, #32 - mov v7.d[0], x7 - mov v7.d[1], x13 - rev64 v7.16b, v7.16b - rbit v7.16b, v7.16b - eor v4.16b, v4.16b, v7.16b - pmull v7.1q, v4.1d, v5.1d - pmull2 v8.1q, v4.2d, v5.2d - ext v10.16b, v4.16b, v4.16b, #8 - pmull v9.1q, v10.1d, v5.1d - pmull2 v10.1q, v10.2d, v5.2d - eor v9.16b, v9.16b, v10.16b - ext v10.16b, v7.16b, v8.16b, #8 - pmull2 v8.1q, v8.2d, v6.2d - eor3 v10.16b, v10.16b, v8.16b, v9.16b - pmull2 v9.1q, v10.2d, v6.2d - mov v7.d[1], v10.d[0] - eor v4.16b, v7.16b, v9.16b - rbit v4.16b, v4.16b - mov w8, v4.s[3] + mov V7.D[0], x7 + mov V7.D[1], x13 + rev64 V7.16B, V7.16B + rbit V7.16B, V7.16B + eor V4.16B, V4.16B, V7.16B + pmull V7.1Q, V4.1D, V5.1D + pmull2 V8.1Q, V4.2D, V5.2D + ext8 V10.16B, V4.16B, V4.16B, #8 + pmull V9.1Q, V10.1D, V5.1D + pmull2 V10.1Q, V10.2D, V5.2D + eor V9.16B, V9.16B, V10.16B + ext8 V10.16B, V7.16B, V8.16B, #8 + pmull2 V8.1Q, V8.2D, V6.2D + eor3 V10.16B, V10.16B, V8.16B, V9.16B + pmull2 V9.1Q, V10.2D, V6.2D + mov V7.D[1], V10.D[0] + eor V4.16B, V7.16B, V9.16B + rbit V4.16B, V4.16B + mov w8, V4.S[3] rev w8, w8 L_aes_gcm_init_arm64_crypto_eor3_done_nonce - st1 {v4.2d}, [x5] - ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b - aesmc v4.16b, v4.16b - aese v4.16b, v9.16b - aesmc v4.16b, v4.16b - aese v4.16b, v10.16b - aesmc v4.16b, v4.16b - ld1 {v7.2d, v8.2d, v9.2d, v10.2d}, [x0], #0x40 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b - aesmc v4.16b, v4.16b - aese v4.16b, v9.16b - aesmc v4.16b, v4.16b - aese v4.16b, v10.16b - aesmc v4.16b, v4.16b + st1 {V4.2D}, [x5] + ld1 {V7.2D, V8.2D, V9.2D, V10.2D}, [x0], #0x40 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B + aesmc V4.16B, V4.16B + aese V4.16B, V9.16B + aesmc V4.16B, V4.16B + aese V4.16B, V10.16B + aesmc V4.16B, V4.16B + ld1 {V7.2D, V8.2D, V9.2D, V10.2D}, [x0], #0x40 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B + aesmc V4.16B, V4.16B + aese V4.16B, V9.16B + aesmc V4.16B, V4.16B + aese V4.16B, V10.16B + aesmc V4.16B, V4.16B subs w1, w1, #10 - ld1 {v7.2d, v8.2d}, [x0], #32 - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + ld1 {V7.2D, V8.2D}, [x0], #32 + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B beq L_aes_gcm_init_arm64_crypto_eor3_round_done - ld1 {v7.2d, v8.2d}, [x0], #32 + ld1 {V7.2D, V8.2D}, [x0], #32 subs w1, w1, #2 - aesmc v4.16b, v4.16b - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + aesmc V4.16B, V4.16B + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B beq L_aes_gcm_init_arm64_crypto_eor3_round_done - ld1 {v7.2d, v8.2d}, [x0], #32 - aesmc v4.16b, v4.16b - aese v4.16b, v7.16b - aesmc v4.16b, v4.16b - aese v4.16b, v8.16b + ld1 {V7.2D, V8.2D}, [x0], #32 + aesmc V4.16B, V4.16B + aese V4.16B, V7.16B + aesmc V4.16B, V4.16B + aese V4.16B, V8.16B L_aes_gcm_init_arm64_crypto_eor3_round_done - ld1 {v7.2d}, [x0] - eor v4.16b, v4.16b, v7.16b - st1 {v4.2d}, [x6] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] + ld1 {V7.2D}, [x0] + eor V4.16B, V4.16B, V7.16B + st1 {V4.2D}, [x6] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] ldp x29, x30, [sp], #48 ret ENDP @@ -32714,31 +32714,31 @@ L_aes_gcm_init_arm64_crypto_eor3_round_done AES_GCM_ghash_block_AARCH64_EOR3 PROC stp x29, x30, [sp, #-32]! add x29, sp, #0 - stp d8, d9, [x29, #16] - ld1 {v6.2d}, [x1] - movi v7.16b, #0x87 - ld1 {v5.2d}, [x2] - ushr v7.2d, v7.2d, #56 - ld1 {v4.2d}, [x0] - rbit v4.16b, v4.16b - eor v8.16b, v6.16b, v4.16b + stp D8, D9, [x29, #16] + ld1 {V6.2D}, [x1] + movi V7.16B, #0x87 + ld1 {V5.2D}, [x2] + ushr V7.2D, V7.2D, #56 + ld1 {V4.2D}, [x0] + rbit V4.16B, V4.16B + eor V8.16B, V6.16B, V4.16B ; X = C * H^1 - pmull v0.1q, v8.1d, v5.1d - pmull2 v1.1q, v8.2d, v5.2d - ext v3.16b, v8.16b, v8.16b, #8 - pmull v2.1q, v3.1d, v5.1d - pmull2 v3.1q, v3.2d, v5.2d - eor v2.16b, v2.16b, v3.16b + pmull V0.1Q, V8.1D, V5.1D + pmull2 V1.1Q, V8.2D, V5.2D + ext8 V3.16B, V8.16B, V8.16B, #8 + pmull V2.1Q, V3.1D, V5.1D + pmull2 V3.1Q, V3.2D, V5.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v7.2d - eor3 v3.16b, v3.16b, v1.16b, v2.16b - pmull2 v2.1q, v3.2d, v7.2d - mov v0.d[1], v3.d[0] - eor v6.16b, v0.16b, v2.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V7.2D + eor3 V3.16B, V3.16B, V1.16B, V2.16B + pmull2 V2.1Q, V3.2D, V7.2D + mov V0.D[1], V3.D[0] + eor V6.16B, V0.16B, V2.16B ; Done GHASH - st1 {v6.2d}, [x1] - ldp d8, d9, [x29, #16] + st1 {V6.2D}, [x1] + ldp D8, D9, [x29, #16] ldp x29, x30, [sp], #32 ret ENDP @@ -32748,99 +32748,99 @@ AES_GCM_ghash_block_AARCH64_EOR3 PROC AES_GCM_aad_update_AARCH64_EOR3 PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] - ld1 {v20.2d}, [x2] - movi v21.16b, #0x87 - ld1 {v12.2d}, [x3] - ushr v21.2d, v21.2d, #56 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + ld1 {V20.2D}, [x2] + movi V21.16B, #0x87 + ld1 {V12.2D}, [x3] + ushr V21.2D, V21.2D, #56 cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done ; Square H => H^2 - pmull2 v11.1q, v12.2d, v12.2d - pmull v10.1q, v12.1d, v12.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v13.16b, v10.16b, v11.16b + pmull2 V11.1Q, V12.2D, V12.2D + pmull V10.1Q, V12.1D, V12.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V13.16B, V10.16B, V11.16B cmp x1, #0x100 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done ; Multiply H and H^2 => H^3 - pmull v8.1q, v12.1d, v13.1d - pmull2 v9.1q, v12.2d, v13.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v13.1d - pmull2 v11.1q, v11.2d, v13.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V13.1D + pmull2 V9.1Q, V12.2D, V13.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V13.1D + pmull2 V11.1Q, V11.2D, V13.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v14.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V14.16B, V8.16B, V10.16B ; Square H^2 => H^4 - pmull2 v11.1q, v13.2d, v13.2d - pmull v10.1q, v13.1d, v13.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v15.16b, v10.16b, v11.16b + pmull2 V11.1Q, V13.2D, V13.2D + pmull V10.1Q, V13.1D, V13.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V15.16B, V10.16B, V11.16B ; Done cmp x1, #0x400 blt L_aes_gcm_aad_update_arm64_crypto_eor3_h_done ; Multiply H and H^4 => H^5 - pmull v8.1q, v12.1d, v15.1d - pmull2 v9.1q, v12.2d, v15.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v15.1d - pmull2 v11.1q, v11.2d, v15.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V15.1D + pmull2 V9.1Q, V12.2D, V15.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V15.1D + pmull2 V11.1Q, V11.2D, V15.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v16.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V16.16B, V8.16B, V10.16B ; Square H^3 => H^6 - pmull2 v11.1q, v14.2d, v14.2d - pmull v10.1q, v14.1d, v14.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v17.16b, v10.16b, v11.16b + pmull2 V11.1Q, V14.2D, V14.2D + pmull V10.1Q, V14.1D, V14.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V17.16B, V10.16B, V11.16B ; Multiply H and H^6 => H^7 - pmull v8.1q, v12.1d, v17.1d - pmull2 v9.1q, v12.2d, v17.2d - ext v11.16b, v12.16b, v12.16b, #8 - pmull v10.1q, v11.1d, v17.1d - pmull2 v11.1q, v11.2d, v17.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V12.1D, V17.1D + pmull2 V9.1Q, V12.2D, V17.2D + ext8 V11.16B, V12.16B, V12.16B, #8 + pmull V10.1Q, V11.1D, V17.1D + pmull2 V11.1Q, V11.2D, V17.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v18.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V18.16B, V8.16B, V10.16B ; Square H^4 => H^8 - pmull2 v11.1q, v15.2d, v15.2d - pmull v10.1q, v15.1d, v15.1d - pmull2 v8.1q, v11.2d, v21.2d - ext v9.16b, v10.16b, v11.16b, #8 - eor v9.16b, v9.16b, v8.16b - pmull2 v11.1q, v9.2d, v21.2d - mov v10.d[1], v9.d[0] - eor v19.16b, v10.16b, v11.16b + pmull2 V11.1Q, V15.2D, V15.2D + pmull V10.1Q, V15.1D, V15.1D + pmull2 V8.1Q, V11.2D, V21.2D + ext8 V9.16B, V10.16B, V11.16B, #8 + eor V9.16B, V9.16B, V8.16B + pmull2 V11.1Q, V9.2D, V21.2D + mov V10.D[1], V9.D[0] + eor V19.16B, V10.16B, V11.16B ; Done L_aes_gcm_aad_update_arm64_crypto_eor3_h_done lsr x1, x1, #4 @@ -32851,94 +32851,94 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_h_done cmp x1, #0x40 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_4 L_aes_gcm_aad_update_arm64_crypto_eor3_start_8 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - rbit v4.16b, v4.16b - rbit v5.16b, v5.16b - rbit v6.16b, v6.16b - rbit v7.16b, v7.16b - eor v0.16b, v0.16b, v20.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + rbit V4.16B, V4.16B + rbit V5.16B, V5.16B + rbit V6.16B, V6.16B + rbit V7.16B, V7.16B + eor V0.16B, V0.16B, V20.16B ; X = C * H^1 - pmull v8.1q, v7.1d, v12.1d - pmull2 v9.1q, v7.2d, v12.2d - ext v11.16b, v7.16b, v7.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V7.1D, V12.1D + pmull2 V9.1Q, V7.2D, V12.2D + ext8 V11.16B, V7.16B, V7.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v6.1d - pmull2 v20.1q, v13.2d, v6.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v6.16b, v6.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V13.1D, V6.1D + pmull2 V20.1Q, V13.2D, V6.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V6.16B, V6.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^3 - pmull v11.1q, v14.1d, v5.1d - pmull2 v20.1q, v14.2d, v5.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v5.16b, v5.16b, #8 - pmull v11.1q, v20.1d, v14.1d - pmull2 v20.1q, v20.2d, v14.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V14.1D, V5.1D + pmull2 V20.1Q, V14.2D, V5.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V5.16B, V5.16B, #8 + pmull V11.1Q, V20.1D, V14.1D + pmull2 V20.1Q, V20.2D, V14.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^4 - pmull v11.1q, v15.1d, v4.1d - pmull2 v20.1q, v15.2d, v4.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v4.16b, v4.16b, #8 - pmull v11.1q, v20.1d, v15.1d - pmull2 v20.1q, v20.2d, v15.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V15.1D, V4.1D + pmull2 V20.1Q, V15.2D, V4.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V4.16B, V4.16B, #8 + pmull V11.1Q, V20.1D, V15.1D + pmull2 V20.1Q, V20.2D, V15.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^5 - pmull v11.1q, v16.1d, v3.1d - pmull2 v20.1q, v16.2d, v3.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v3.16b, v3.16b, #8 - pmull v11.1q, v20.1d, v16.1d - pmull2 v20.1q, v20.2d, v16.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V16.1D, V3.1D + pmull2 V20.1Q, V16.2D, V3.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V3.16B, V3.16B, #8 + pmull V11.1Q, V20.1D, V16.1D + pmull2 V20.1Q, V20.2D, V16.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^6 - pmull v11.1q, v17.1d, v2.1d - pmull2 v20.1q, v17.2d, v2.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v2.16b, v2.16b, #8 - pmull v11.1q, v20.1d, v17.1d - pmull2 v20.1q, v20.2d, v17.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V17.1D, V2.1D + pmull2 V20.1Q, V17.2D, V2.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V2.16B, V2.16B, #8 + pmull V11.1Q, V20.1D, V17.1D + pmull2 V20.1Q, V20.2D, V17.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^7 - pmull v11.1q, v18.1d, v1.1d - pmull2 v20.1q, v18.2d, v1.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v1.16b, v1.16b, #8 - pmull v11.1q, v20.1d, v18.1d - pmull2 v20.1q, v20.2d, v18.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V18.1D, V1.1D + pmull2 V20.1Q, V18.2D, V1.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V1.16B, V1.16B, #8 + pmull V11.1Q, V20.1D, V18.1D + pmull2 V20.1Q, V20.2D, V18.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^8 - pmull v11.1q, v19.1d, v0.1d - pmull2 v20.1q, v19.2d, v0.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v0.16b, v0.16b, #8 - pmull v11.1q, v20.1d, v19.1d - pmull2 v20.1q, v20.2d, v19.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V19.1D, V0.1D + pmull2 V20.1Q, V19.2D, V0.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V0.16B, V0.16B, #8 + pmull V11.1Q, V20.1D, V19.1D + pmull2 V20.1Q, V20.2D, V19.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #8 cmp x1, #8 @@ -32949,53 +32949,53 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_start_8 cmp x1, #16 blt L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 L_aes_gcm_aad_update_arm64_crypto_eor3_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v0.16b, v0.16b, v20.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V0.16B, V0.16B, V20.16B ; X = C * H^1 - pmull v8.1q, v3.1d, v12.1d - pmull2 v9.1q, v3.2d, v12.2d - ext v11.16b, v3.16b, v3.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V3.1D, V12.1D + pmull2 V9.1Q, V3.2D, V12.2D + ext8 V11.16B, V3.16B, V3.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v2.1d - pmull2 v20.1q, v13.2d, v2.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v2.16b, v2.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V13.1D, V2.1D + pmull2 V20.1Q, V13.2D, V2.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V2.16B, V2.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^3 - pmull v11.1q, v14.1d, v1.1d - pmull2 v20.1q, v14.2d, v1.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v1.16b, v1.16b, #8 - pmull v11.1q, v20.1d, v14.1d - pmull2 v20.1q, v20.2d, v14.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V14.1D, V1.1D + pmull2 V20.1Q, V14.2D, V1.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V1.16B, V1.16B, #8 + pmull V11.1Q, V20.1D, V14.1D + pmull2 V20.1Q, V20.2D, V14.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; X += C * H^4 - pmull v11.1q, v15.1d, v0.1d - pmull2 v20.1q, v15.2d, v0.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v0.16b, v0.16b, #8 - pmull v11.1q, v20.1d, v15.1d - pmull2 v20.1q, v20.2d, v15.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V15.1D, V0.1D + pmull2 V20.1Q, V15.2D, V0.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V0.16B, V0.16B, #8 + pmull V11.1Q, V20.1D, V15.1D + pmull2 V20.1Q, V20.2D, V15.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #4 cmp x1, #4 @@ -33004,33 +33004,33 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_start_4 blt L_aes_gcm_aad_update_arm64_crypto_eor3_done beq L_aes_gcm_aad_update_arm64_crypto_eor3_start_1 L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 - ld1 {v0.16b, v1.16b}, [x0], #32 - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - eor v3.16b, v20.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + eor V3.16B, V20.16B, V0.16B ; X = C * H^1 - pmull v8.1q, v1.1d, v12.1d - pmull2 v9.1q, v1.2d, v12.2d - ext v11.16b, v1.16b, v1.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V1.1D, V12.1D + pmull2 V9.1Q, V1.2D, V12.2D + ext8 V11.16B, V1.16B, V1.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; X += C * H^2 - pmull v11.1q, v13.1d, v3.1d - pmull2 v20.1q, v13.2d, v3.2d - eor v8.16b, v8.16b, v11.16b - eor v9.16b, v9.16b, v20.16b - ext v20.16b, v3.16b, v3.16b, #8 - pmull v11.1q, v20.1d, v13.1d - pmull2 v20.1q, v20.2d, v13.2d - eor3 v10.16b, v10.16b, v20.16b, v11.16b + pmull V11.1Q, V13.1D, V3.1D + pmull2 V20.1Q, V13.2D, V3.2D + eor V8.16B, V8.16B, V11.16B + eor V9.16B, V9.16B, V20.16B + ext8 V20.16B, V3.16B, V3.16B, #8 + pmull V11.1Q, V20.1D, V13.1D + pmull2 V20.1Q, V20.2D, V13.2D + eor3 V10.16B, V10.16B, V20.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH sub x1, x1, #2 cmp x1, #1 @@ -33039,32 +33039,32 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_start_2 L_aes_gcm_aad_update_arm64_crypto_eor3_start_1 cbz x1, L_aes_gcm_aad_update_arm64_crypto_eor3_done L_aes_gcm_aad_update_arm64_crypto_eor3_both_1 - ld1 {v0.16b}, [x0], #16 - rbit v0.16b, v0.16b - eor v3.16b, v20.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + rbit V0.16B, V0.16B + eor V3.16B, V20.16B, V0.16B ; X = C * H^1 - pmull v8.1q, v3.1d, v12.1d - pmull2 v9.1q, v3.2d, v12.2d - ext v11.16b, v3.16b, v3.16b, #8 - pmull v10.1q, v11.1d, v12.1d - pmull2 v11.1q, v11.2d, v12.2d - eor v10.16b, v10.16b, v11.16b + pmull V8.1Q, V3.1D, V12.1D + pmull2 V9.1Q, V3.2D, V12.2D + ext8 V11.16B, V3.16B, V3.16B, #8 + pmull V10.1Q, V11.1D, V12.1D + pmull2 V11.1Q, V11.2D, V12.2D + eor V10.16B, V10.16B, V11.16B ; Reduce - ext v11.16b, v8.16b, v9.16b, #8 - pmull2 v9.1q, v9.2d, v21.2d - eor3 v11.16b, v11.16b, v9.16b, v10.16b - pmull2 v10.1q, v11.2d, v21.2d - mov v8.d[1], v11.d[0] - eor v20.16b, v8.16b, v10.16b + ext8 V11.16B, V8.16B, V9.16B, #8 + pmull2 V9.1Q, V9.2D, V21.2D + eor3 V11.16B, V11.16B, V9.16B, V10.16B + pmull2 V10.1Q, V11.2D, V21.2D + mov V8.D[1], V11.D[0] + eor V20.16B, V8.16B, V10.16B ; Done GHASH subs x1, x1, #1 bne L_aes_gcm_aad_update_arm64_crypto_eor3_both_1 L_aes_gcm_aad_update_arm64_crypto_eor3_done - st1 {v20.2d}, [x2] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st1 {V20.2D}, [x2] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -33072,55 +33072,55 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_encrypt_block_AARCH64_EOR3 AES_GCM_encrypt_block_AARCH64_EOR3 PROC - ld1 {v5.2d}, [x4] - ld1 {v4.2d}, [x3] - mov w5, v5.s[3] + ld1 {V5.2D}, [x4] + ld1 {V4.2D}, [x3] + mov w5, V5.S[3] rev w5, w5 add w5, w5, #1 rev w5, w5 - mov v5.s[3], w5 - st1 {v5.2d}, [x4] - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b - aesmc v5.16b, v5.16b - aese v5.16b, v2.16b - aesmc v5.16b, v5.16b - aese v5.16b, v3.16b - aesmc v5.16b, v5.16b - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b - aesmc v5.16b, v5.16b - aese v5.16b, v2.16b - aesmc v5.16b, v5.16b - aese v5.16b, v3.16b - aesmc v5.16b, v5.16b + mov V5.S[3], w5 + st1 {V5.2D}, [x4] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B + aesmc V5.16B, V5.16B + aese V5.16B, V2.16B + aesmc V5.16B, V5.16B + aese V5.16B, V3.16B + aesmc V5.16B, V5.16B + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B + aesmc V5.16B, V5.16B + aese V5.16B, V2.16B + aesmc V5.16B, V5.16B + aese V5.16B, V3.16B + aesmc V5.16B, V5.16B subs w1, w1, #10 - ld1 {v0.2d, v1.2d}, [x0], #32 - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + ld1 {V0.2D, V1.2D}, [x0], #32 + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B beq L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done - ld1 {v0.2d, v1.2d}, [x0], #32 + ld1 {V0.2D, V1.2D}, [x0], #32 subs w1, w1, #2 - aesmc v5.16b, v5.16b - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + aesmc V5.16B, V5.16B + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B beq L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done - ld1 {v0.2d, v1.2d}, [x0], #32 - aesmc v5.16b, v5.16b - aese v5.16b, v0.16b - aesmc v5.16b, v5.16b - aese v5.16b, v1.16b + ld1 {V0.2D, V1.2D}, [x0], #32 + aesmc V5.16B, V5.16B + aese V5.16B, V0.16B + aesmc V5.16B, V5.16B + aese V5.16B, V1.16B L_aes_gcm_encrypt_block_arm64_crypto_eor3_round_done - ld1 {v0.2d}, [x0] - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v5.16b - st1 {v4.2d}, [x2] + ld1 {V0.2D}, [x0] + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V5.16B + st1 {V4.2D}, [x2] ret ENDP AREA |.text|, CODE, READONLY @@ -33130,102 +33130,102 @@ AES_GCM_encrypt_update_AARCH64_EOR3 PROC stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] - stp d8, d9, [x29, #32] - stp d10, d11, [x29, #48] - stp d12, d13, [x29, #64] - stp d14, d15, [x29, #80] - ld1 {v13.2d}, [x7] - movi v27.16b, #0x87 - ld1 {v26.2d}, [x5] - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x6] - mov w9, v13.s[3] + stp D8, D9, [x29, #32] + stp D10, D11, [x29, #48] + stp D12, D13, [x29, #64] + stp D14, D15, [x29, #80] + ld1 {V13.2D}, [x7] + movi V27.16B, #0x87 + ld1 {V26.2D}, [x5] + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x6] + mov w9, V13.S[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w4, #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w4, #0x200 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done lsr w8, w4, #4 @@ -33237,23 +33237,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_h_done cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -33262,1168 +33262,1168 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_192_done ENDIF @@ -34434,23 +34434,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_256 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -34459,1285 +34459,1285 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0], #16 cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x0, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x0, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_256_done ENDIF @@ -35748,23 +35748,23 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_start_128 cmp w8, #32 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 rev w16, w16 rev w15, w15 @@ -35773,1069 +35773,1069 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_8 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v20.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v21.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v0.16b}, [x3], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v1.16b}, [x3], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v2.16b}, [x3], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - ld1 {v3.16b}, [x3], #16 - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - ld1 {v13.2d}, [x7] - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V20.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V21.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V0.16B}, [x3], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V1.16B}, [x3], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V2.16B}, [x3], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + ld1 {V3.16B}, [x3], #16 + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + ld1 {V13.2D}, [x7] + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w16, w16 - rbit v20.16b, v20.16b + rbit V20.16B, V20.16B rev w15, w15 - rbit v21.16b, v21.16b + rbit V21.16B, V21.16B rev w14, w14 - rbit v0.16b, v0.16b + rbit V0.16B, V0.16B rev w13, w13 - rbit v1.16b, v1.16b + rbit V1.16B, V1.16B rev w12, w12 - rbit v2.16b, v2.16b + rbit V2.16B, V2.16B rev w11, w11 - rbit v3.16b, v3.16b + rbit V3.16B, V3.16B rev w10, w9 - mov v14.s[3], w17 - mov v15.s[3], w16 - mov v16.s[3], w15 - mov v17.s[3], w14 - mov v8.s[3], w13 - mov v9.s[3], w12 - mov v10.s[3], w11 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w17 + mov V15.S[3], w16 + mov V16.S[3], w15 + mov V17.S[3], w14 + mov V8.S[3], w13 + mov V9.S[3], w12 + mov V10.S[3], w11 + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v30.16b, v30.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V30.16B, V30.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - mov v28.d[1], v31.d[0] - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor v26.16b, v28.16b, v30.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + mov V28.D[1], V31.D[0] + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor V26.16B, V28.16B, V30.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B ; Done GHASH - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v18.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2], #0x40 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V18.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_8 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d}, [x0], #32 - ld1 {v10.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D}, [x0], #32 + ld1 {V10.2D}, [x0] cmp w8, #1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1 cmp w8, #4 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 rev w12, w12 rev w11, w11 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - ld1 {v19.16b}, [x3], #16 - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - ld1 {v21.16b}, [x3], #16 - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + ld1 {V19.16B}, [x3], #16 + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + ld1 {V21.16B}, [x3], #16 + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 rev w12, w12 rev w11, w11 - rbit v19.16b, v19.16b + rbit V19.16B, V19.16B rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w12 - mov v16.s[3], w11 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - rbit v20.16b, v20.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - rbit v21.16b, v21.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - eor v18.16b, v18.16b, v26.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b + mov V14.S[3], w13 + mov V15.S[3], w12 + mov V16.S[3], w11 + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + rbit V20.16B, V20.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + rbit V21.16B, V21.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + eor V18.16B, V18.16B, V26.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v30.16b, v30.16b, v31.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V30.16B, V30.16B, V31.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - eor v20.16b, v20.16b, v16.16b - eor v21.16b, v21.16b, v17.16b + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + eor V20.16B, V20.16B, V16.16B + eor V21.16B, V21.16B, V17.16B cmp w8, #4 - st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x2], #0x40 + st1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x2], #0x40 bge L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_both_4 L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 rev w10, w9 - mov v14.s[3], w13 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V14.S[3], w13 + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v18.16b, v18.16b, v14.16b - eor v19.16b, v19.16b, v15.16b - st1 {v18.16b, v19.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V18.16B, V18.16B, V14.16B + eor V19.16B, V19.16B, V15.16B + st1 {V18.16B, V19.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x3], #16 - eor v18.16b, v18.16b, v14.16b - st1 {v18.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x3], #16 + eor V18.16B, V18.16B, V14.16B + st1 {V18.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_encrypt_update_arm64_crypto_eor3_128_done ENDIF L_aes_gcm_encrypt_update_arm64_crypto_eor3_done rev w9, w9 - mov v13.s[3], w9 - st1 {v26.2d}, [x5] - st1 {v13.2d}, [x7] + mov V13.S[3], w9 + st1 {V26.2D}, [x5] + st1 {V13.2D}, [x7] ldr x17, [x29, #24] - ldp d8, d9, [x29, #32] - ldp d10, d11, [x29, #48] - ldp d12, d13, [x29, #64] - ldp d14, d15, [x29, #80] + ldp D8, D9, [x29, #32] + ldp D10, D11, [x29, #48] + ldp D12, D13, [x29, #64] + ldp D14, D15, [x29, #80] ldp x29, x30, [sp], #0x60 ret ENDP @@ -36843,39 +36843,39 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_encrypt_final_AARCH64_EOR3 AES_GCM_encrypt_final_AARCH64_EOR3 PROC - ld1 {v5.2d}, [x0] - movi v6.16b, #0x87 - ld1 {v4.2d}, [x5] - ushr v6.2d, v6.2d, #56 - ld1 {v7.2d}, [x6] + ld1 {V5.2D}, [x0] + movi V6.16B, #0x87 + ld1 {V4.2D}, [x5] + ushr V6.2D, V6.2D, #56 + ld1 {V7.2D}, [x6] ubfiz x4, x4, #3, #32 rbit x4, x4 - mov v0.d[0], x4 + mov V0.D[0], x4 ubfiz x3, x3, #3, #32 rbit x3, x3 - mov v0.d[1], x3 - eor v5.16b, v5.16b, v0.16b - pmull v0.1q, v5.1d, v4.1d - pmull2 v1.1q, v5.2d, v4.2d - ext v3.16b, v5.16b, v5.16b, #8 - pmull v2.1q, v3.1d, v4.1d - pmull2 v3.1q, v3.2d, v4.2d - eor v2.16b, v2.16b, v3.16b + mov V0.D[1], x3 + eor V5.16B, V5.16B, V0.16B + pmull V0.1Q, V5.1D, V4.1D + pmull2 V1.1Q, V5.2D, V4.2D + ext8 V3.16B, V5.16B, V5.16B, #8 + pmull V2.1Q, V3.1D, V4.1D + pmull2 V3.1Q, V3.2D, V4.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v6.2d - eor3 v3.16b, v3.16b, v1.16b, v2.16b - pmull2 v2.1q, v3.2d, v6.2d - mov v0.d[1], v3.d[0] - eor v5.16b, v0.16b, v2.16b - rbit v5.16b, v5.16b - eor v5.16b, v5.16b, v7.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V6.2D + eor3 V3.16B, V3.16B, V1.16B, V2.16B + pmull2 V2.1Q, V3.2D, V6.2D + mov V0.D[1], V3.D[0] + eor V5.16B, V0.16B, V2.16B + rbit V5.16B, V5.16B + eor V5.16B, V5.16B, V7.16B cmp w2, #16 bne L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_partial - st1 {v5.16b}, [x1] + st1 {V5.16B}, [x1] b L_aes_gcm_encrypt_final_arm64_crypto_eor3_done L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_partial - st1 {v5.16b}, [x0] + st1 {V5.16B}, [x0] cmp w2, #8 blt L_aes_gcm_encrypt_final_arm64_crypto_eor3_tag_start_dw ldr x8, [x0], #8 @@ -36910,102 +36910,102 @@ AES_GCM_decrypt_update_AARCH64_EOR3 PROC stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] - stp d8, d9, [x29, #32] - stp d10, d11, [x29, #48] - stp d12, d13, [x29, #64] - stp d14, d15, [x29, #80] - ld1 {v13.2d}, [x7] - movi v27.16b, #0x87 - ld1 {v26.2d}, [x5] - ushr v27.2d, v27.2d, #56 - ld1 {v22.2d}, [x6] - mov w9, v13.s[3] + stp D8, D9, [x29, #32] + stp D10, D11, [x29, #48] + stp D12, D13, [x29, #64] + stp D14, D15, [x29, #80] + ld1 {V13.2D}, [x7] + movi V27.16B, #0x87 + ld1 {V26.2D}, [x5] + ushr V27.2D, V27.2D, #56 + ld1 {V22.2D}, [x6] + mov w9, V13.S[3] rev w9, w9 cmp w4, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done ; Square H => H^2 - pmull2 v31.1q, v22.2d, v22.2d - pmull v30.1q, v22.1d, v22.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v23.16b, v30.16b, v31.16b + pmull2 V31.1Q, V22.2D, V22.2D + pmull V30.1Q, V22.1D, V22.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V23.16B, V30.16B, V31.16B cmp w4, #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done ; Multiply H and H^2 => H^3 - pmull v28.1q, v22.1d, v23.1d - pmull2 v29.1q, v22.2d, v23.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v23.1d - pmull2 v31.1q, v31.2d, v23.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V23.1D + pmull2 V29.1Q, V22.2D, V23.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V23.1D + pmull2 V31.1Q, V31.2D, V23.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v24.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V24.16B, V28.16B, V30.16B ; Square H^2 => H^4 - pmull2 v31.1q, v23.2d, v23.2d - pmull v30.1q, v23.1d, v23.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v25.16b, v30.16b, v31.16b + pmull2 V31.1Q, V23.2D, V23.2D + pmull V30.1Q, V23.1D, V23.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V25.16B, V30.16B, V31.16B ; Done cmp w4, #0x200 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done ; Multiply H and H^4 => H^5 - pmull v28.1q, v22.1d, v25.1d - pmull2 v29.1q, v22.2d, v25.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v25.1d - pmull2 v31.1q, v31.2d, v25.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V25.1D + pmull2 V29.1Q, V22.2D, V25.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V25.1D + pmull2 V31.1Q, V31.2D, V25.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v4.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V4.16B, V28.16B, V30.16B ; Square H^3 => H^6 - pmull2 v31.1q, v24.2d, v24.2d - pmull v30.1q, v24.1d, v24.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v5.16b, v30.16b, v31.16b + pmull2 V31.1Q, V24.2D, V24.2D + pmull V30.1Q, V24.1D, V24.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V5.16B, V30.16B, V31.16B ; Multiply H and H^6 => H^7 - pmull v28.1q, v22.1d, v5.1d - pmull2 v29.1q, v22.2d, v5.2d - ext v31.16b, v22.16b, v22.16b, #8 - pmull v30.1q, v31.1d, v5.1d - pmull2 v31.1q, v31.2d, v5.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V22.1D, V5.1D + pmull2 V29.1Q, V22.2D, V5.2D + ext8 V31.16B, V22.16B, V22.16B, #8 + pmull V30.1Q, V31.1D, V5.1D + pmull2 V31.1Q, V31.2D, V5.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v6.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V6.16B, V28.16B, V30.16B ; Square H^4 => H^8 - pmull2 v31.1q, v25.2d, v25.2d - pmull v30.1q, v25.1d, v25.1d - pmull2 v28.1q, v31.2d, v27.2d - ext v29.16b, v30.16b, v31.16b, #8 - eor v29.16b, v29.16b, v28.16b - pmull2 v31.1q, v29.2d, v27.2d - mov v30.d[1], v29.d[0] - eor v7.16b, v30.16b, v31.16b + pmull2 V31.1Q, V25.2D, V25.2D + pmull V30.1Q, V25.1D, V25.1D + pmull2 V28.1Q, V31.2D, V27.2D + ext8 V29.16B, V30.16B, V31.16B, #8 + eor V29.16B, V29.16B, V28.16B + pmull2 V31.1Q, V29.2D, V27.2D + mov V30.D[1], V29.D[0] + eor V7.16B, V30.16B, V31.16B ; Done L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done lsr w8, w4, #4 @@ -37017,1195 +37017,1195 @@ L_aes_gcm_decrypt_update_arm64_crypto_eor3_h_done cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v11.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v11.16b - eor v17.16b, v17.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V11.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V11.16B + eor V17.16B, V17.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v11.16b - eor v15.16b, v15.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V11.16B + eor V15.16B, V15.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_start_1 - ld1 {v15.16b}, [x3], #16 + ld1 {V15.16B}, [x3], #16 add w9, w9, #1 - mov v14.16b, v13.16b - rbit v15.16b, v15.16b + mov V14.16B, V13.16B + rbit V15.16B, V15.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v16.16b, v26.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V16.16B, V26.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v16.1d, v22.1d - pmull2 v29.1q, v16.2d, v22.2d - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - ext v31.16b, v16.16b, v16.16b, #8 - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b + pmull V28.1Q, V16.1D, V22.1D + pmull2 V29.1Q, V16.2D, V22.2D + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + ext8 V31.16B, V16.16B, V16.16B, #8 + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - mov v28.d[1], v31.d[0] - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v14.16b, v11.16b - eor v14.16b, v14.16b, v12.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + mov V28.D[1], V31.D[0] + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V14.16B, V11.16B + eor V14.16B, V14.16B, V12.16B ; Done GHASH - rbit v15.16b, v15.16b - eor v14.16b, v14.16b, v15.16b - st1 {v14.16b}, [x2], #16 + rbit V15.16B, V15.16B + eor V14.16B, V14.16B, V15.16B + st1 {V14.16B}, [x2], #16 L_aes_gcm_decrypt_update_arm64_crypto_eor3_192_done ENDIF b L_aes_gcm_decrypt_update_arm64_crypto_eor3_done @@ -38215,1310 +38215,1310 @@ L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_256 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #176] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #192] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #208] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #224] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #176] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #192] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #208] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #224] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x0], #0x40 - ld1 {v12.2d}, [x0], #16 + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x0], #0x40 + ld1 {V12.2D}, [x0], #16 cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v16.16b, v9.16b - aesmc v16.16b, v16.16b - aese v17.16b, v9.16b - aesmc v17.16b, v17.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v16.16b, v10.16b - aesmc v16.16b, v16.16b - aese v17.16b, v10.16b - aesmc v17.16b, v17.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - aese v16.16b, v11.16b - aesmc v16.16b, v16.16b - aese v17.16b, v11.16b - aesmc v17.16b, v17.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - aese v16.16b, v29.16b - eor v16.16b, v16.16b, v30.16b - aese v17.16b, v29.16b - eor v17.16b, v17.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V16.16B, V9.16B + aesmc V16.16B, V16.16B + aese V17.16B, V9.16B + aesmc V17.16B, V17.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V16.16B, V10.16B + aesmc V16.16B, V16.16B + aese V17.16B, V10.16B + aesmc V17.16B, V17.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + aese V16.16B, V11.16B + aesmc V16.16B, V16.16B + aese V17.16B, V11.16B + aesmc V17.16B, V17.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + aese V16.16B, V29.16B + eor V16.16B, V16.16B, V30.16B + aese V17.16B, V29.16B + eor V17.16B, V17.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - aesmc v15.16b, v15.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - aese v15.16b, v10.16b - aesmc v15.16b, v15.16b - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - aese v15.16b, v11.16b - aesmc v15.16b, v15.16b - ld1 {v29.2d, v30.2d}, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - aese v15.16b, v29.16b - eor v15.16b, v15.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + aesmc V15.16B, V15.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + aese V15.16B, V10.16B + aesmc V15.16B, V15.16B + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + aese V15.16B, V11.16B + aesmc V15.16B, V15.16B + ld1 {V29.2D, V30.2D}, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + aese V15.16B, V29.16B + eor V15.16B, V15.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - aesmc v14.16b, v14.16b - aese v14.16b, v10.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v14.16b, v11.16b - aesmc v14.16b, v14.16b - ldr q29, [x0] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ldr q30, [x0, #16] - aese v14.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + aesmc V14.16B, V14.16B + aese V14.16B, V10.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V14.16B, V11.16B + aesmc V14.16B, V14.16B + ldr Q29, [x0] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ldr Q30, [x0, #16] + aese V14.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_256_done ENDIF @@ -39529,1094 +39529,1094 @@ L_aes_gcm_decrypt_update_arm64_crypto_eor3_start_128 cmp w8, #32 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b + mov V11.16B, V13.16B rev w17, w17 - mov v14.s[3], w17 + mov V14.S[3], w17 rev w16, w16 - mov v15.s[3], w16 + mov V15.S[3], w16 rev w15, w15 - mov v16.s[3], w15 + mov V16.S[3], w15 rev w14, w14 - mov v17.s[3], w14 + mov V17.S[3], w14 rev w13, w13 - mov v8.s[3], w13 + mov V8.S[3], w13 rev w12, w12 - mov v9.s[3], w12 + mov V9.S[3], w12 rev w11, w11 - mov v10.s[3], w11 + mov V10.S[3], w11 rev w10, w9 - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v18.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v19.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v20.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v21.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - ld1 {v0.16b}, [x3], #16 - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - ld1 {v1.16b}, [x3], #16 - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - ld1 {v2.16b}, [x3], #16 - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - ld1 {v3.16b}, [x3], #16 - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - ld1 {v13.2d}, [x7] - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V18.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V19.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V20.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V21.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + ld1 {V0.16B}, [x3], #16 + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + ld1 {V1.16B}, [x3], #16 + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + ld1 {V2.16B}, [x3], #16 + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + ld1 {V3.16B}, [x3], #16 + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + ld1 {V13.2D}, [x7] + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_8 - ldr q12, [x0] + ldr Q12, [x0] add w17, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w16, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w15, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w14, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B add w13, w9, #5 - mov v8.16b, v13.16b + mov V8.16B, V13.16B add w12, w9, #6 - mov v9.16b, v13.16b + mov V9.16B, V13.16B add w11, w9, #7 - mov v10.16b, v13.16b + mov V10.16B, V13.16B add w9, w9, #8 - mov v11.16b, v13.16b - rbit v18.16b, v18.16b + mov V11.16B, V13.16B + rbit V18.16B, V18.16B rev w17, w17 - rbit v19.16b, v19.16b - mov v14.s[3], w17 + rbit V19.16B, V19.16B + mov V14.S[3], w17 rev w16, w16 - rbit v20.16b, v20.16b - mov v15.s[3], w16 + rbit V20.16B, V20.16B + mov V15.S[3], w16 rev w15, w15 - rbit v21.16b, v21.16b - mov v16.s[3], w15 + rbit V21.16B, V21.16B + mov V16.S[3], w15 rev w14, w14 - rbit v0.16b, v0.16b - mov v17.s[3], w14 + rbit V0.16B, V0.16B + mov V17.S[3], w14 rev w13, w13 - rbit v1.16b, v1.16b - mov v8.s[3], w13 + rbit V1.16B, V1.16B + mov V8.S[3], w13 rev w12, w12 - rbit v2.16b, v2.16b - mov v9.s[3], w12 + rbit V2.16B, V2.16B + mov V9.S[3], w12 rev w11, w11 - rbit v3.16b, v3.16b - mov v10.s[3], w11 + rbit V3.16B, V3.16B + mov V10.S[3], w11 rev w10, w9 - eor v18.16b, v18.16b, v26.16b - mov v11.s[3], w10 - ldr q13, [x0, #16] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + eor V18.16B, V18.16B, V26.16B + mov V11.S[3], w10 + ldr Q13, [x0, #16] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ext v31.16b, v3.16b, v3.16b, #8 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v30.16b, v30.16b, v31.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ext8 V31.16B, V3.16B, V3.16B, #8 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V30.16B, V30.16B, V31.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - ext v26.16b, v2.16b, v2.16b, #8 - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #32] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #32] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ext v26.16b, v1.16b, v1.16b, #8 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #48] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #48] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ext v26.16b, v0.16b, v0.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #64] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #64] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - ext v26.16b, v21.16b, v21.16b, #8 - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #80] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #80] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #96] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #96] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #112] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #112] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B subs w8, w8, #8 - ldr q12, [x0, #128] - aese v14.16b, v13.16b - aesmc v14.16b, v14.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v15.16b, v13.16b - aesmc v15.16b, v15.16b - mov v28.d[1], v31.d[0] - aese v16.16b, v13.16b - aesmc v16.16b, v16.16b - eor v26.16b, v28.16b, v30.16b - aese v17.16b, v13.16b - aesmc v17.16b, v17.16b + ldr Q12, [x0, #128] + aese V14.16B, V13.16B + aesmc V14.16B, V14.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V15.16B, V13.16B + aesmc V15.16B, V15.16B + mov V28.D[1], V31.D[0] + aese V16.16B, V13.16B + aesmc V16.16B, V16.16B + eor V26.16B, V28.16B, V30.16B + aese V17.16B, V13.16B + aesmc V17.16B, V17.16B ; Done GHASH - aese v8.16b, v13.16b - aesmc v8.16b, v8.16b - ld1 {v18.16b}, [x3], #16 - aese v9.16b, v13.16b - aesmc v9.16b, v9.16b - ld1 {v19.16b}, [x3], #16 - aese v10.16b, v13.16b - aesmc v10.16b, v10.16b - ld1 {v20.16b}, [x3], #16 - aese v11.16b, v13.16b - aesmc v11.16b, v11.16b - ldr q13, [x0, #144] - aese v14.16b, v12.16b - aesmc v14.16b, v14.16b - ld1 {v21.16b}, [x3], #16 - aese v15.16b, v12.16b - aesmc v15.16b, v15.16b - ld1 {v0.16b}, [x3], #16 - aese v16.16b, v12.16b - aesmc v16.16b, v16.16b - ld1 {v1.16b}, [x3], #16 - aese v17.16b, v12.16b - aesmc v17.16b, v17.16b - ld1 {v2.16b}, [x3], #16 - aese v8.16b, v12.16b - aesmc v8.16b, v8.16b - ld1 {v3.16b}, [x3], #16 - aese v9.16b, v12.16b - aesmc v9.16b, v9.16b - aese v10.16b, v12.16b - aesmc v10.16b, v10.16b - aese v11.16b, v12.16b - aesmc v11.16b, v11.16b - ldr q12, [x0, #160] - aese v14.16b, v13.16b - eor v14.16b, v14.16b, v12.16b - aese v15.16b, v13.16b - eor v15.16b, v15.16b, v12.16b - aese v16.16b, v13.16b - eor v16.16b, v16.16b, v12.16b - aese v17.16b, v13.16b - eor v17.16b, v17.16b, v12.16b - aese v8.16b, v13.16b - eor v8.16b, v8.16b, v12.16b - aese v9.16b, v13.16b - eor v9.16b, v9.16b, v12.16b - aese v10.16b, v13.16b - eor v10.16b, v10.16b, v12.16b - aese v11.16b, v13.16b - eor v11.16b, v11.16b, v12.16b - ld1 {v13.2d}, [x7] - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [x2], #0x40 + aese V8.16B, V13.16B + aesmc V8.16B, V8.16B + ld1 {V18.16B}, [x3], #16 + aese V9.16B, V13.16B + aesmc V9.16B, V9.16B + ld1 {V19.16B}, [x3], #16 + aese V10.16B, V13.16B + aesmc V10.16B, V10.16B + ld1 {V20.16B}, [x3], #16 + aese V11.16B, V13.16B + aesmc V11.16B, V11.16B + ldr Q13, [x0, #144] + aese V14.16B, V12.16B + aesmc V14.16B, V14.16B + ld1 {V21.16B}, [x3], #16 + aese V15.16B, V12.16B + aesmc V15.16B, V15.16B + ld1 {V0.16B}, [x3], #16 + aese V16.16B, V12.16B + aesmc V16.16B, V16.16B + ld1 {V1.16B}, [x3], #16 + aese V17.16B, V12.16B + aesmc V17.16B, V17.16B + ld1 {V2.16B}, [x3], #16 + aese V8.16B, V12.16B + aesmc V8.16B, V8.16B + ld1 {V3.16B}, [x3], #16 + aese V9.16B, V12.16B + aesmc V9.16B, V9.16B + aese V10.16B, V12.16B + aesmc V10.16B, V10.16B + aese V11.16B, V12.16B + aesmc V11.16B, V11.16B + ldr Q12, [x0, #160] + aese V14.16B, V13.16B + eor V14.16B, V14.16B, V12.16B + aese V15.16B, V13.16B + eor V15.16B, V15.16B, V12.16B + aese V16.16B, V13.16B + eor V16.16B, V16.16B, V12.16B + aese V17.16B, V13.16B + eor V17.16B, V17.16B, V12.16B + aese V8.16B, V13.16B + eor V8.16B, V8.16B, V12.16B + aese V9.16B, V13.16B + eor V9.16B, V9.16B, V12.16B + aese V10.16B, V13.16B + eor V10.16B, V10.16B, V12.16B + aese V11.16B, V13.16B + eor V11.16B, V11.16B, V12.16B + ld1 {V13.2D}, [x7] + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + st1 {V8.16B, V9.16B, V10.16B, V11.16B}, [x2], #0x40 cmp w8, #8 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_8 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_8 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - rbit v0.16b, v0.16b - rbit v1.16b, v1.16b - rbit v2.16b, v2.16b - rbit v3.16b, v3.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + rbit V0.16B, V0.16B + rbit V1.16B, V1.16B + rbit V2.16B, V2.16B + rbit V3.16B, V3.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v3.1d, v22.1d - pmull2 v29.1q, v3.2d, v22.2d - ext v31.16b, v3.16b, v3.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V3.1D, V22.1D + pmull2 V29.1Q, V3.2D, V22.2D + ext8 V31.16B, V3.16B, V3.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v2.1d - pmull2 v26.1q, v23.2d, v2.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v2.16b, v2.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V2.1D + pmull2 V26.1Q, V23.2D, V2.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V2.16B, V2.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v1.1d - pmull2 v26.1q, v24.2d, v1.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v1.16b, v1.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V1.1D + pmull2 V26.1Q, V24.2D, V1.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V1.16B, V1.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v0.1d - pmull2 v26.1q, v25.2d, v0.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v0.16b, v0.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V0.1D + pmull2 V26.1Q, V25.2D, V0.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V0.16B, V0.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^5 - pmull v31.1q, v4.1d, v21.1d - pmull2 v26.1q, v4.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v4.1d - pmull2 v26.1q, v26.2d, v4.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V4.1D, V21.1D + pmull2 V26.1Q, V4.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V4.1D + pmull2 V26.1Q, V26.2D, V4.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^6 - pmull v31.1q, v5.1d, v20.1d - pmull2 v26.1q, v5.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v5.1d - pmull2 v26.1q, v26.2d, v5.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V5.1D, V20.1D + pmull2 V26.1Q, V5.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V5.1D + pmull2 V26.1Q, V26.2D, V5.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^7 - pmull v31.1q, v6.1d, v19.1d - pmull2 v26.1q, v6.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v6.1d - pmull2 v26.1q, v26.2d, v6.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V6.1D, V19.1D + pmull2 V26.1Q, V6.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V6.1D + pmull2 V26.1Q, V26.2D, V6.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^8 - pmull v31.1q, v7.1d, v18.1d - pmull2 v26.1q, v7.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v7.1d - pmull2 v26.1q, v26.2d, v7.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V7.1D, V18.1D + pmull2 V26.1Q, V7.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V7.1D + pmull2 V26.1Q, V26.2D, V7.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_4 - ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x0], #0x40 - ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x0], #0x40 - ld1 {v8.2d, v9.2d}, [x0], #32 - ld1 {v10.2d}, [x0] + ld1 {V0.2D, V1.2D, V2.2D, V3.2D}, [x0], #0x40 + ld1 {V4.2D, V5.2D, V6.2D, V7.2D}, [x0], #0x40 + ld1 {V8.2D, V9.2D}, [x0], #32 + ld1 {V10.2D}, [x0] cmp w8, #1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1 cmp w8, #4 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b + mov V17.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w12, w12 - mov v15.s[3], w12 + mov V15.S[3], w12 rev w11, w11 - mov v16.s[3], w11 + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - ld1 {v18.16b}, [x3], #16 - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - ld1 {v19.16b}, [x3], #16 - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - ld1 {v20.16b}, [x3], #16 - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - ld1 {v21.16b}, [x3], #16 - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + ld1 {V18.16B}, [x3], #16 + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + ld1 {V19.16B}, [x3], #16 + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + ld1 {V20.16B}, [x3], #16 + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + ld1 {V21.16B}, [x3], #16 + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_4 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w12, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B add w11, w9, #3 - mov v16.16b, v13.16b + mov V16.16B, V13.16B add w9, w9, #4 - mov v17.16b, v13.16b - rbit v18.16b, v18.16b + mov V17.16B, V13.16B + rbit V18.16B, V18.16B rev w13, w13 - rbit v19.16b, v19.16b - mov v14.s[3], w13 + rbit V19.16B, V19.16B + mov V14.S[3], w13 rev w12, w12 - rbit v20.16b, v20.16b - mov v15.s[3], w12 + rbit V20.16B, V20.16B + mov V15.S[3], w12 rev w11, w11 - rbit v21.16b, v21.16b - mov v16.s[3], w11 + rbit V21.16B, V21.16B + mov V16.S[3], w11 rev w10, w9 - mov v17.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - eor v18.16b, v18.16b, v26.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b + mov V17.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + eor V18.16B, V18.16B, V26.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - aese v16.16b, v0.16b - aesmc v16.16b, v16.16b - ext v31.16b, v21.16b, v21.16b, #8 - aese v17.16b, v0.16b - aesmc v17.16b, v17.16b - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - eor v30.16b, v30.16b, v31.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + aese V16.16B, V0.16B + aesmc V16.16B, V16.16B + ext8 V31.16B, V21.16B, V21.16B, #8 + aese V17.16B, V0.16B + aesmc V17.16B, V17.16B + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + eor V30.16B, V30.16B, V31.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - aese v16.16b, v1.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v1.16b - aesmc v17.16b, v17.16b - ext v26.16b, v20.16b, v20.16b, #8 - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v2.16b - aesmc v16.16b, v16.16b - aese v17.16b, v2.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + aese V16.16B, V1.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V1.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V2.16B + aesmc V16.16B, V16.16B + aese V17.16B, V2.16B + aesmc V17.16B, V17.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - ext v26.16b, v19.16b, v19.16b, #8 - aese v16.16b, v3.16b - aesmc v16.16b, v16.16b - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - aese v17.16b, v3.16b - aesmc v17.16b, v17.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + aese V16.16B, V3.16B + aesmc V16.16B, V16.16B + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + aese V17.16B, V3.16B + aesmc V17.16B, V17.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - aese v16.16b, v4.16b - aesmc v16.16b, v16.16b - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - aese v17.16b, v4.16b - aesmc v17.16b, v17.16b - ext v26.16b, v18.16b, v18.16b, #8 - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - eor3 v30.16b, v30.16b, v26.16b, v31.16b - aese v16.16b, v5.16b - aesmc v16.16b, v16.16b - aese v17.16b, v5.16b - aesmc v17.16b, v17.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + aese V16.16B, V4.16B + aesmc V16.16B, V16.16B + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + aese V17.16B, V4.16B + aesmc V17.16B, V17.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + eor3 V30.16B, V30.16B, V26.16B, V31.16B + aese V16.16B, V5.16B + aesmc V16.16B, V16.16B + aese V17.16B, V5.16B + aesmc V17.16B, V17.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - pmull2 v29.1q, v29.2d, v27.2d - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b - eor3 v31.16b, v31.16b, v29.16b, v30.16b - aese v16.16b, v6.16b - aesmc v16.16b, v16.16b - pmull2 v30.1q, v31.2d, v27.2d - aese v17.16b, v6.16b - aesmc v17.16b, v17.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + pmull2 V29.1Q, V29.2D, V27.2D + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B + eor3 V31.16B, V31.16B, V29.16B, V30.16B + aese V16.16B, V6.16B + aesmc V16.16B, V16.16B + pmull2 V30.1Q, V31.2D, V27.2D + aese V17.16B, V6.16B + aesmc V17.16B, V17.16B subs w8, w8, #4 - mov v28.d[1], v31.d[0] - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - eor v26.16b, v28.16b, v30.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b + mov V28.D[1], V31.D[0] + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + eor V26.16B, V28.16B, V30.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B ; Done GHASH - aese v16.16b, v7.16b - aesmc v16.16b, v16.16b - aese v17.16b, v7.16b - aesmc v17.16b, v17.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x3], #0x40 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v16.16b, v8.16b - aesmc v16.16b, v16.16b - aese v17.16b, v8.16b - aesmc v17.16b, v17.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - aese v16.16b, v9.16b - eor v16.16b, v16.16b, v10.16b - aese v17.16b, v9.16b - eor v17.16b, v17.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - eor v16.16b, v16.16b, v20.16b - eor v17.16b, v17.16b, v21.16b + aese V16.16B, V7.16B + aesmc V16.16B, V16.16B + aese V17.16B, V7.16B + aesmc V17.16B, V17.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x3], #0x40 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V16.16B, V8.16B + aesmc V16.16B, V16.16B + aese V17.16B, V8.16B + aesmc V17.16B, V17.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + aese V16.16B, V9.16B + eor V16.16B, V16.16B, V10.16B + aese V17.16B, V9.16B + eor V17.16B, V17.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + eor V16.16B, V16.16B, V20.16B + eor V17.16B, V17.16B, V21.16B cmp w8, #4 - st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x2], #0x40 + st1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x2], #0x40 bge L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_both_4 L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_end_4 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - rbit v20.16b, v20.16b - rbit v21.16b, v21.16b - eor v18.16b, v18.16b, v26.16b + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + rbit V20.16B, V20.16B + rbit V21.16B, V21.16B + eor V18.16B, V18.16B, V26.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v20.1d - pmull2 v26.1q, v23.2d, v20.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v20.16b, v20.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V20.1D + pmull2 V26.1Q, V23.2D, V20.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V20.16B, V20.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^3 - pmull v31.1q, v24.1d, v19.1d - pmull2 v26.1q, v24.2d, v19.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v19.16b, v19.16b, #8 - pmull v31.1q, v26.1d, v24.1d - pmull2 v26.1q, v26.2d, v24.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V24.1D, V19.1D + pmull2 V26.1Q, V24.2D, V19.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V19.16B, V19.16B, #8 + pmull V31.1Q, V26.1D, V24.1D + pmull2 V26.1Q, V26.2D, V24.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; X += C * H^4 - pmull v31.1q, v25.1d, v18.1d - pmull2 v26.1q, v25.2d, v18.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v18.16b, v18.16b, #8 - pmull v31.1q, v26.1d, v25.1d - pmull2 v26.1q, v26.2d, v25.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V25.1D, V18.1D + pmull2 V26.1Q, V25.2D, V18.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V18.16B, V18.16B, #8 + pmull V31.1Q, V26.1D, V25.1D + pmull2 V26.1Q, V26.2D, V25.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cmp w8, #1 beq L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1 blt L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_2 add w13, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B add w9, w9, #2 - mov v15.16b, v13.16b + mov V15.16B, V13.16B rev w13, w13 - mov v14.s[3], w13 + mov V14.S[3], w13 rev w10, w9 - mov v15.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v15.16b, v0.16b - aesmc v15.16b, v15.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v15.16b, v1.16b - aesmc v15.16b, v15.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v15.16b, v2.16b - aesmc v15.16b, v15.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v15.16b, v3.16b - aesmc v15.16b, v15.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v15.16b, v4.16b - aesmc v15.16b, v15.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v15.16b, v5.16b - aesmc v15.16b, v15.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v15.16b, v6.16b - aesmc v15.16b, v15.16b + mov V15.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V15.16B, V0.16B + aesmc V15.16B, V15.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V15.16B, V1.16B + aesmc V15.16B, V15.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V15.16B, V2.16B + aesmc V15.16B, V15.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V15.16B, V3.16B + aesmc V15.16B, V15.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V15.16B, V4.16B + aesmc V15.16B, V15.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V15.16B, V5.16B + aesmc V15.16B, V15.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V15.16B, V6.16B + aesmc V15.16B, V15.16B subs w8, w8, #2 - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v15.16b, v7.16b - aesmc v15.16b, v15.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - ld1 {v18.16b}, [x3], #16 - aese v15.16b, v8.16b - aesmc v15.16b, v15.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v19.16b}, [x3], #16 - aese v15.16b, v9.16b - eor v15.16b, v15.16b, v10.16b - eor v14.16b, v14.16b, v18.16b - eor v15.16b, v15.16b, v19.16b - st1 {v14.16b, v15.16b}, [x2], #32 - rbit v18.16b, v18.16b - rbit v19.16b, v19.16b - eor v21.16b, v26.16b, v18.16b + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V15.16B, V7.16B + aesmc V15.16B, V15.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + ld1 {V18.16B}, [x3], #16 + aese V15.16B, V8.16B + aesmc V15.16B, V15.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V19.16B}, [x3], #16 + aese V15.16B, V9.16B + eor V15.16B, V15.16B, V10.16B + eor V14.16B, V14.16B, V18.16B + eor V15.16B, V15.16B, V19.16B + st1 {V14.16B, V15.16B}, [x2], #32 + rbit V18.16B, V18.16B + rbit V19.16B, V19.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v19.1d, v22.1d - pmull2 v29.1q, v19.2d, v22.2d - ext v31.16b, v19.16b, v19.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V19.1D, V22.1D + pmull2 V29.1Q, V19.2D, V22.2D + ext8 V31.16B, V19.16B, V19.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; X += C * H^2 - pmull v31.1q, v23.1d, v21.1d - pmull2 v26.1q, v23.2d, v21.2d - eor v28.16b, v28.16b, v31.16b - eor v29.16b, v29.16b, v26.16b - ext v26.16b, v21.16b, v21.16b, #8 - pmull v31.1q, v26.1d, v23.1d - pmull2 v26.1q, v26.2d, v23.2d - eor3 v30.16b, v30.16b, v26.16b, v31.16b + pmull V31.1Q, V23.1D, V21.1D + pmull2 V26.1Q, V23.2D, V21.2D + eor V28.16B, V28.16B, V31.16B + eor V29.16B, V29.16B, V26.16B + ext8 V26.16B, V21.16B, V21.16B, #8 + pmull V31.1Q, V26.1D, V23.1D + pmull2 V26.1Q, V26.2D, V23.2D + eor3 V30.16B, V30.16B, V26.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH cbz w8, L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_start_1 add w9, w9, #1 - mov v14.16b, v13.16b + mov V14.16B, V13.16B rev w10, w9 - mov v14.s[3], w10 - aese v14.16b, v0.16b - aesmc v14.16b, v14.16b - aese v14.16b, v1.16b - aesmc v14.16b, v14.16b - aese v14.16b, v2.16b - aesmc v14.16b, v14.16b - aese v14.16b, v3.16b - aesmc v14.16b, v14.16b - aese v14.16b, v4.16b - aesmc v14.16b, v14.16b - aese v14.16b, v5.16b - aesmc v14.16b, v14.16b - aese v14.16b, v6.16b - aesmc v14.16b, v14.16b - aese v14.16b, v7.16b - aesmc v14.16b, v14.16b - aese v14.16b, v8.16b - aesmc v14.16b, v14.16b - aese v14.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - ld1 {v18.16b}, [x3], #16 - eor v14.16b, v14.16b, v18.16b - st1 {v14.16b}, [x2], #16 - rbit v18.16b, v18.16b - eor v21.16b, v26.16b, v18.16b + mov V14.S[3], w10 + aese V14.16B, V0.16B + aesmc V14.16B, V14.16B + aese V14.16B, V1.16B + aesmc V14.16B, V14.16B + aese V14.16B, V2.16B + aesmc V14.16B, V14.16B + aese V14.16B, V3.16B + aesmc V14.16B, V14.16B + aese V14.16B, V4.16B + aesmc V14.16B, V14.16B + aese V14.16B, V5.16B + aesmc V14.16B, V14.16B + aese V14.16B, V6.16B + aesmc V14.16B, V14.16B + aese V14.16B, V7.16B + aesmc V14.16B, V14.16B + aese V14.16B, V8.16B + aesmc V14.16B, V14.16B + aese V14.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + ld1 {V18.16B}, [x3], #16 + eor V14.16B, V14.16B, V18.16B + st1 {V14.16B}, [x2], #16 + rbit V18.16B, V18.16B + eor V21.16B, V26.16B, V18.16B ; X = C * H^1 - pmull v28.1q, v21.1d, v22.1d - pmull2 v29.1q, v21.2d, v22.2d - ext v31.16b, v21.16b, v21.16b, #8 - pmull v30.1q, v31.1d, v22.1d - pmull2 v31.1q, v31.2d, v22.2d - eor v30.16b, v30.16b, v31.16b + pmull V28.1Q, V21.1D, V22.1D + pmull2 V29.1Q, V21.2D, V22.2D + ext8 V31.16B, V21.16B, V21.16B, #8 + pmull V30.1Q, V31.1D, V22.1D + pmull2 V31.1Q, V31.2D, V22.2D + eor V30.16B, V30.16B, V31.16B ; Reduce - ext v31.16b, v28.16b, v29.16b, #8 - pmull2 v29.1q, v29.2d, v27.2d - eor3 v31.16b, v31.16b, v29.16b, v30.16b - pmull2 v30.1q, v31.2d, v27.2d - mov v28.d[1], v31.d[0] - eor v26.16b, v28.16b, v30.16b + ext8 V31.16B, V28.16B, V29.16B, #8 + pmull2 V29.1Q, V29.2D, V27.2D + eor3 V31.16B, V31.16B, V29.16B, V30.16B + pmull2 V30.1Q, V31.2D, V27.2D + mov V28.D[1], V31.D[0] + eor V26.16B, V28.16B, V30.16B ; Done GHASH L_aes_gcm_decrypt_update_arm64_crypto_eor3_128_done ENDIF L_aes_gcm_decrypt_update_arm64_crypto_eor3_done rev w9, w9 - mov v13.s[3], w9 - st1 {v26.2d}, [x5] - st1 {v13.2d}, [x7] + mov V13.S[3], w9 + st1 {V26.2D}, [x5] + st1 {V13.2D}, [x7] ldr x17, [x29, #24] - ldp d8, d9, [x29, #32] - ldp d10, d11, [x29, #48] - ldp d12, d13, [x29, #64] - ldp d14, d15, [x29, #80] + ldp D8, D9, [x29, #32] + ldp D10, D11, [x29, #48] + ldp D12, D13, [x29, #64] + ldp D14, D15, [x29, #80] ldp x29, x30, [sp], #0x60 ret ENDP @@ -40624,42 +40624,42 @@ L_aes_gcm_decrypt_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_decrypt_final_AARCH64_EOR3 AES_GCM_decrypt_final_AARCH64_EOR3 PROC - ld1 {v5.2d}, [x0] - movi v6.16b, #0x87 - ld1 {v4.2d}, [x5] - ushr v6.2d, v6.2d, #56 - ld1 {v7.2d}, [x6] + ld1 {V5.2D}, [x0] + movi V6.16B, #0x87 + ld1 {V4.2D}, [x5] + ushr V6.2D, V6.2D, #56 + ld1 {V7.2D}, [x6] ubfiz x4, x4, #3, #32 rbit x4, x4 - mov v0.d[0], x4 + mov V0.D[0], x4 ubfiz x3, x3, #3, #32 rbit x3, x3 - mov v0.d[1], x3 - eor v5.16b, v5.16b, v0.16b - pmull v0.1q, v5.1d, v4.1d - pmull2 v1.1q, v5.2d, v4.2d - ext v3.16b, v5.16b, v5.16b, #8 - pmull v2.1q, v3.1d, v4.1d - pmull2 v3.1q, v3.2d, v4.2d - eor v2.16b, v2.16b, v3.16b + mov V0.D[1], x3 + eor V5.16B, V5.16B, V0.16B + pmull V0.1Q, V5.1D, V4.1D + pmull2 V1.1Q, V5.2D, V4.2D + ext8 V3.16B, V5.16B, V5.16B, #8 + pmull V2.1Q, V3.1D, V4.1D + pmull2 V3.1Q, V3.2D, V4.2D + eor V2.16B, V2.16B, V3.16B ; Reduce - ext v3.16b, v0.16b, v1.16b, #8 - pmull2 v1.1q, v1.2d, v6.2d - eor3 v3.16b, v3.16b, v1.16b, v2.16b - pmull2 v2.1q, v3.2d, v6.2d - mov v0.d[1], v3.d[0] - eor v5.16b, v0.16b, v2.16b - rbit v5.16b, v5.16b - eor v5.16b, v5.16b, v7.16b + ext8 V3.16B, V0.16B, V1.16B, #8 + pmull2 V1.1Q, V1.2D, V6.2D + eor3 V3.16B, V3.16B, V1.16B, V2.16B + pmull2 V2.1Q, V3.2D, V6.2D + mov V0.D[1], V3.D[0] + eor V5.16B, V0.16B, V2.16B + rbit V5.16B, V5.16B + eor V5.16B, V5.16B, V7.16B cmp w2, #16 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag - ld1 {v0.16b}, [x1] + ld1 {V0.16B}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag ubfiz x2, x2, #0, #32 - eor v0.16b, v0.16b, v0.16b + eor V0.16B, V0.16B, V0.16B mov x10, x2 - st1 {v0.2d}, [x0] + st1 {V0.2D}, [x0] cmp x10, #8 blt L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_dw ldr x9, [x1], #8 @@ -40685,9 +40685,9 @@ L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_byte bne L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_start_byte L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_end_bytes sub x0, x0, x2 - ld1 {v0.2d}, [x0] + ld1 {V0.2D}, [x0] mov x10, #16 - st1 {v5.2d}, [x0] + st1 {V5.2D}, [x0] sub x10, x10, x2 add x0, x0, x2 L_aes_gcm_decrypt_final_arm64_crypto_eor3_calc_tag_byte @@ -40695,11 +40695,11 @@ L_aes_gcm_decrypt_final_arm64_crypto_eor3_calc_tag_byte subs x10, x10, #1 bne L_aes_gcm_decrypt_final_arm64_crypto_eor3_calc_tag_byte subs x0, x0, #16 - ld1 {v5.2d}, [x0] + ld1 {V5.2D}, [x0] L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded - eor v0.16b, v0.16b, v5.16b - mov x9, v0.d[0] - mov x10, v0.d[1] + eor V0.16B, V0.16B, V5.16B + mov x9, V0.D[0] + mov x10, V0.D[1] mov w11, #-180 orr x9, x9, x10 cmp x9, #0 @@ -40720,9 +40720,9 @@ AES_XTS_encrypt_AARCH64 PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #24] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x5], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x5], #0x40 - ld1 {v4.16b}, [x3] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x5], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x5], #0x40 + ld1 {V4.16B}, [x3] lsr w8, w2, #4 and w2, w2, #15 mov x19, #0x87 @@ -40731,38 +40731,38 @@ AES_XTS_encrypt_AARCH64 PROC bgt L_aes_xts_encrypt_arm64_crypto_start_256 ; AES_XTS_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 - ld1 {v28.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v4.16b, v27.16b - eor v4.16b, v4.16b, v28.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x5], #0x40 + ld1 {V28.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V4.16B, V27.16B + eor V4.16B, V4.16B, V28.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -40775,243 +40775,243 @@ AES_XTS_encrypt_AARCH64 PROC cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_192_start_2 L_aes_xts_encrypt_arm64_crypto_192_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x17, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B extr x11, x17, x16, #63 - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B and x9, x19, x11, asr 63 - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B extr x15, x13, x12, #63 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B and x9, x19, x15, asr 63 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B extr x17, x15, x14, #63 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aese v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aese v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aese v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aese V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aese V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aese V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_192_start_4 L_aes_xts_encrypt_arm64_crypto_192_start_2 cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_192_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B extr x11, x13, x12, #63 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aese v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aese V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_192_start_1 cbz w8, L_aes_xts_encrypt_arm64_crypto_192_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x11, x11, x10, #63 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_192_done cbz w2, L_aes_xts_encrypt_arm64_crypto_192_partial_done sub x1, x1, #16 - ld1 {v0.16b}, [x1], #16 - st1 {v0.2d}, [x6] + ld1 {V0.16B}, [x1], #16 + st1 {V0.2D}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_192_start_byte ldrb w12, [x6] @@ -41023,78 +41023,78 @@ L_aes_xts_encrypt_arm64_crypto_192_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_encrypt_arm64_crypto_192_partial_done ENDIF b L_aes_xts_encrypt_arm64_crypto_done ; AES_XTS_256 L_aes_xts_encrypt_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 - ld1 {v28.2d, v29.2d}, [x5], #32 - ld1 {v30.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b - aese v4.16b, v29.16b - eor v4.16b, v4.16b, v30.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d, v29.2d}, [x4], #32 - ld1 {v30.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x5], #0x40 + ld1 {V28.2D, V29.2D}, [x5], #32 + ld1 {V30.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V4.16B, V27.16B + aesmc V4.16B, V4.16B + aese V4.16B, V28.16B + aesmc V4.16B, V4.16B + aese V4.16B, V29.16B + eor V4.16B, V4.16B, V30.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D, V29.2D}, [x4], #32 + ld1 {V30.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -41107,271 +41107,271 @@ L_aes_xts_encrypt_arm64_crypto_start_256 cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_256_start_2 L_aes_xts_encrypt_arm64_crypto_256_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x17, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B extr x11, x17, x16, #63 - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B and x9, x19, x11, asr 63 - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B extr x15, x13, x12, #63 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B and x9, x19, x15, asr 63 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B extr x17, x15, x14, #63 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v2.16b, v25.16b - aesmc v2.16b, v2.16b - aese v3.16b, v25.16b - aesmc v3.16b, v3.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v2.16b, v26.16b - aesmc v2.16b, v2.16b - aese v3.16b, v26.16b - aesmc v3.16b, v3.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b - aese v2.16b, v27.16b - aesmc v2.16b, v2.16b - aese v3.16b, v27.16b - aesmc v3.16b, v3.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b - aese v2.16b, v28.16b - aesmc v2.16b, v2.16b - aese v3.16b, v28.16b - aesmc v3.16b, v3.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aese v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aese v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aese v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V2.16B, V25.16B + aesmc V2.16B, V2.16B + aese V3.16B, V25.16B + aesmc V3.16B, V3.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V2.16B, V26.16B + aesmc V2.16B, V2.16B + aese V3.16B, V26.16B + aesmc V3.16B, V3.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V1.16B, V27.16B + aesmc V1.16B, V1.16B + aese V2.16B, V27.16B + aesmc V2.16B, V2.16B + aese V3.16B, V27.16B + aesmc V3.16B, V3.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V1.16B, V28.16B + aesmc V1.16B, V1.16B + aese V2.16B, V28.16B + aesmc V2.16B, V2.16B + aese V3.16B, V28.16B + aesmc V3.16B, V3.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aese V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aese V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aese V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_256_start_4 L_aes_xts_encrypt_arm64_crypto_256_start_2 cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_256_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B extr x11, x13, x12, #63 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v1.16b, v25.16b - aesmc v1.16b, v1.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v1.16b, v26.16b - aesmc v1.16b, v1.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v1.16b, v27.16b - aesmc v1.16b, v1.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v1.16b, v28.16b - aesmc v1.16b, v1.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aese v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V1.16B, V25.16B + aesmc V1.16B, V1.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V1.16B, V26.16B + aesmc V1.16B, V1.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V1.16B, V27.16B + aesmc V1.16B, V1.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V1.16B, V28.16B + aesmc V1.16B, V1.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aese V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_256_start_1 cbz w8, L_aes_xts_encrypt_arm64_crypto_256_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x11, x11, x10, #63 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_256_done cbz w2, L_aes_xts_encrypt_arm64_crypto_256_partial_done sub x1, x1, #16 - ld1 {v0.16b}, [x1], #16 - st1 {v0.2d}, [x6] + ld1 {V0.16B}, [x1], #16 + st1 {V0.2D}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_256_start_byte ldrb w12, [x6] @@ -41383,72 +41383,72 @@ L_aes_xts_encrypt_arm64_crypto_256_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - aesmc v0.16b, v0.16b - aese v0.16b, v26.16b - aesmc v0.16b, v0.16b - aese v0.16b, v27.16b - aesmc v0.16b, v0.16b - aese v0.16b, v28.16b - aesmc v0.16b, v0.16b - aese v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + aesmc V0.16B, V0.16B + aese V0.16B, V26.16B + aesmc V0.16B, V0.16B + aese V0.16B, V27.16B + aesmc V0.16B, V0.16B + aese V0.16B, V28.16B + aesmc V0.16B, V0.16B + aese V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_encrypt_arm64_crypto_256_partial_done ENDIF b L_aes_xts_encrypt_arm64_crypto_done ; AES_XTS_128 L_aes_xts_encrypt_arm64_crypto_start_128 IF :LNOT::DEF:NO_AES_128 - ld1 {v24.2d, v25.2d}, [x5], #32 - ld1 {v26.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - eor v4.16b, v4.16b, v26.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d}, [x4], #32 - ld1 {v26.2d}, [x4] + ld1 {V24.2D, V25.2D}, [x5], #32 + ld1 {V26.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + eor V4.16B, V4.16B, V26.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D}, [x4], #32 + ld1 {V26.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -41461,215 +41461,215 @@ L_aes_xts_encrypt_arm64_crypto_start_128 cmp w8, #4 blt L_aes_xts_encrypt_arm64_crypto_128_start_2 L_aes_xts_encrypt_arm64_crypto_128_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x17, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v2.16b, v16.16b - aesmc v2.16b, v2.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V2.16B, V16.16B + aesmc V2.16B, V2.16B extr x11, x17, x16, #63 - aese v3.16b, v16.16b - aesmc v3.16b, v3.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V3.16B, V16.16B + aesmc V3.16B, V3.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v2.16b, v17.16b - aesmc v2.16b, v2.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V2.16B, V17.16B + aesmc V2.16B, V2.16B and x9, x19, x11, asr 63 - aese v3.16b, v17.16b - aesmc v3.16b, v3.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V3.16B, V17.16B + aesmc V3.16B, V3.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v2.16b, v18.16b - aesmc v2.16b, v2.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V2.16B, V18.16B + aesmc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aese v3.16b, v18.16b - aesmc v3.16b, v3.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V3.16B, V18.16B + aesmc V3.16B, V3.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v2.16b, v19.16b - aesmc v2.16b, v2.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V2.16B, V19.16B + aesmc V2.16B, V2.16B extr x15, x13, x12, #63 - aese v3.16b, v19.16b - aesmc v3.16b, v3.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V3.16B, V19.16B + aesmc V3.16B, V3.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v2.16b, v20.16b - aesmc v2.16b, v2.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V2.16B, V20.16B + aesmc V2.16B, V2.16B and x9, x19, x15, asr 63 - aese v3.16b, v20.16b - aesmc v3.16b, v3.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V3.16B, V20.16B + aesmc V3.16B, V3.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B extr x17, x15, x14, #63 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v2.16b, v21.16b - aesmc v2.16b, v2.16b + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V2.16B, V21.16B + aesmc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aese v3.16b, v21.16b - aesmc v3.16b, v3.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v2.16b, v22.16b - aesmc v2.16b, v2.16b - aese v3.16b, v22.16b - aesmc v3.16b, v3.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v2.16b, v23.16b - aesmc v2.16b, v2.16b - aese v3.16b, v23.16b - aesmc v3.16b, v3.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v2.16b, v24.16b - aesmc v2.16b, v2.16b - aese v3.16b, v24.16b - aesmc v3.16b, v3.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aese v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aese v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aese v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V3.16B, V21.16B + aesmc V3.16B, V3.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V2.16B, V22.16B + aesmc V2.16B, V2.16B + aese V3.16B, V22.16B + aesmc V3.16B, V3.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V2.16B, V23.16B + aesmc V2.16B, V2.16B + aese V3.16B, V23.16B + aesmc V3.16B, V3.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V2.16B, V24.16B + aesmc V2.16B, V2.16B + aese V3.16B, V24.16B + aesmc V3.16B, V3.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aese V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aese V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aese V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_encrypt_arm64_crypto_128_start_4 L_aes_xts_encrypt_arm64_crypto_128_start_2 cmp w8, #2 blt L_aes_xts_encrypt_arm64_crypto_128_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x13, asr 63 - aese v1.16b, v16.16b - aesmc v1.16b, v1.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b + aese V1.16B, V16.16B + aesmc V1.16B, V1.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B extr x11, x13, x12, #63 - aese v1.16b, v17.16b - aesmc v1.16b, v1.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V1.16B, V17.16B + aesmc V1.16B, V1.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aese v1.16b, v18.16b - aesmc v1.16b, v1.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b + aese V1.16B, V18.16B + aesmc V1.16B, V1.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v1.16b, v19.16b - aesmc v1.16b, v1.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V1.16B, V19.16B + aesmc V1.16B, V1.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B extr x13, x11, x10, #63 - aese v1.16b, v20.16b - aesmc v1.16b, v1.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b + aese V1.16B, V20.16B + aesmc V1.16B, V1.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aese v1.16b, v21.16b - aesmc v1.16b, v1.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v1.16b, v22.16b - aesmc v1.16b, v1.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v1.16b, v23.16b - aesmc v1.16b, v1.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v1.16b, v24.16b - aesmc v1.16b, v1.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aese v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aese V1.16B, V21.16B + aesmc V1.16B, V1.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V1.16B, V22.16B + aesmc V1.16B, V1.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V1.16B, V23.16B + aesmc V1.16B, V1.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V1.16B, V24.16B + aesmc V1.16B, V1.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aese V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_encrypt_arm64_crypto_128_start_1 cbz w8, L_aes_xts_encrypt_arm64_crypto_128_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B and x9, x19, x11, asr 63 - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B extr x11, x11, x10, #63 - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_encrypt_arm64_crypto_128_done cbz w2, L_aes_xts_encrypt_arm64_crypto_128_partial_done sub x1, x1, #16 - ld1 {v0.16b}, [x1], #16 - st1 {v0.2d}, [x6] + ld1 {V0.16B}, [x1], #16 + st1 {V0.2D}, [x6] mov w9, w2 L_aes_xts_encrypt_arm64_crypto_128_start_byte ldrb w12, [x6] @@ -41681,30 +41681,30 @@ L_aes_xts_encrypt_arm64_crypto_128_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aese v0.16b, v16.16b - aesmc v0.16b, v0.16b - aese v0.16b, v17.16b - aesmc v0.16b, v0.16b - aese v0.16b, v18.16b - aesmc v0.16b, v0.16b - aese v0.16b, v19.16b - aesmc v0.16b, v0.16b - aese v0.16b, v20.16b - aesmc v0.16b, v0.16b - aese v0.16b, v21.16b - aesmc v0.16b, v0.16b - aese v0.16b, v22.16b - aesmc v0.16b, v0.16b - aese v0.16b, v23.16b - aesmc v0.16b, v0.16b - aese v0.16b, v24.16b - aesmc v0.16b, v0.16b - aese v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aese V0.16B, V16.16B + aesmc V0.16B, V0.16B + aese V0.16B, V17.16B + aesmc V0.16B, V0.16B + aese V0.16B, V18.16B + aesmc V0.16B, V0.16B + aese V0.16B, V19.16B + aesmc V0.16B, V0.16B + aese V0.16B, V20.16B + aesmc V0.16B, V0.16B + aese V0.16B, V21.16B + aesmc V0.16B, V0.16B + aese V0.16B, V22.16B + aesmc V0.16B, V0.16B + aese V0.16B, V23.16B + aesmc V0.16B, V0.16B + aese V0.16B, V24.16B + aesmc V0.16B, V0.16B + aese V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_encrypt_arm64_crypto_128_partial_done ENDIF L_aes_xts_encrypt_arm64_crypto_done @@ -41720,9 +41720,9 @@ AES_XTS_decrypt_AARCH64 PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 stp x17, x19, [x29, #24] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x5], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x5], #0x40 - ld1 {v4.16b}, [x3] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x5], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x5], #0x40 + ld1 {V4.16B}, [x3] lsr w8, w2, #4 ands w2, w2, #15 mov x19, #0x87 @@ -41733,38 +41733,38 @@ AES_XTS_decrypt_AARCH64 PROC bgt L_aes_xts_decrypt_arm64_crypto_start_256 ; AES_XTS_192 IF :LNOT::DEF:NO_AES_192 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 - ld1 {v28.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v4.16b, v27.16b - eor v4.16b, v4.16b, v28.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x5], #0x40 + ld1 {V28.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V4.16B, V27.16B + eor V4.16B, V4.16B, V28.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -41777,273 +41777,273 @@ AES_XTS_decrypt_AARCH64 PROC cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_192_start_2 L_aes_xts_decrypt_arm64_crypto_192_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x17, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B extr x11, x17, x16, #63 - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B and x9, x19, x11, asr 63 - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B extr x15, x13, x12, #63 - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B and x9, x19, x15, asr 63 - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B extr x17, x15, x14, #63 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - aesd v2.16b, v27.16b - eor v2.16b, v2.16b, v28.16b - aesd v3.16b, v27.16b - eor v3.16b, v3.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + aesd V2.16B, V27.16B + eor V2.16B, V2.16B, V28.16B + aesd V3.16B, V27.16B + eor V3.16B, V3.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_192_start_4 L_aes_xts_decrypt_arm64_crypto_192_start_2 cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_192_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B extr x11, x13, x12, #63 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - aesd v1.16b, v27.16b - eor v1.16b, v1.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + aesd V1.16B, V27.16B + eor V1.16B, V1.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_192_start_1 cbz w8, L_aes_xts_decrypt_arm64_crypto_192_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x11, x11, x10, #63 - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_192_done cbz w2, L_aes_xts_decrypt_arm64_crypto_192_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 - mov v5.d[0], x12 - mov v5.d[1], x13 - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v0.16b, v0.16b, v5.16b - st1 {v0.2d}, [x6] + mov V5.D[0], x12 + mov V5.D[1], x13 + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V0.16B, V0.16B, V5.16B + st1 {V0.2D}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_192_start_byte @@ -42056,78 +42056,78 @@ L_aes_xts_decrypt_arm64_crypto_192_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - eor v0.16b, v0.16b, v28.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + eor V0.16B, V0.16B, V28.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_decrypt_arm64_crypto_192_partial_done ENDIF b L_aes_xts_decrypt_arm64_crypto_done ; AES_XTS_256 L_aes_xts_decrypt_arm64_crypto_start_256 IF :LNOT::DEF:NO_AES_256 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x5], #0x40 - ld1 {v28.2d, v29.2d}, [x5], #32 - ld1 {v30.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - aesmc v4.16b, v4.16b - aese v4.16b, v26.16b - aesmc v4.16b, v4.16b - aese v4.16b, v27.16b - aesmc v4.16b, v4.16b - aese v4.16b, v28.16b - aesmc v4.16b, v4.16b - aese v4.16b, v29.16b - eor v4.16b, v4.16b, v30.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x4], #0x40 - ld1 {v28.2d, v29.2d}, [x4], #32 - ld1 {v30.2d}, [x4] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x5], #0x40 + ld1 {V28.2D, V29.2D}, [x5], #32 + ld1 {V30.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + aesmc V4.16B, V4.16B + aese V4.16B, V26.16B + aesmc V4.16B, V4.16B + aese V4.16B, V27.16B + aesmc V4.16B, V4.16B + aese V4.16B, V28.16B + aesmc V4.16B, V4.16B + aese V4.16B, V29.16B + eor V4.16B, V4.16B, V30.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x4], #0x40 + ld1 {V28.2D, V29.2D}, [x4], #32 + ld1 {V30.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -42140,305 +42140,305 @@ L_aes_xts_decrypt_arm64_crypto_start_256 cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_256_start_2 L_aes_xts_decrypt_arm64_crypto_256_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x17, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B extr x11, x17, x16, #63 - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B and x9, x19, x11, asr 63 - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B extr x15, x13, x12, #63 - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B and x9, x19, x15, asr 63 - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B extr x17, x15, x14, #63 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v25.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v25.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v26.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v26.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v27.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v27.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v28.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v28.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - aesd v2.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - aesd v3.16b, v29.16b - eor v3.16b, v3.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V25.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V25.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V26.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V26.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V27.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V27.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V28.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V28.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + aesd V2.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + aesd V3.16B, V29.16B + eor V3.16B, V3.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_256_start_4 L_aes_xts_decrypt_arm64_crypto_256_start_2 cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_256_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B extr x11, x13, x12, #63 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v25.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v26.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v27.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v28.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - aesd v1.16b, v29.16b - eor v1.16b, v1.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V25.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V26.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V27.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V28.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + aesd V1.16B, V29.16B + eor V1.16B, V1.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_256_start_1 cbz w8, L_aes_xts_decrypt_arm64_crypto_256_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x11, x11, x10, #63 - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_256_done cbz w2, L_aes_xts_decrypt_arm64_crypto_256_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 - mov v5.d[0], x12 - mov v5.d[1], x13 - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v0.16b, v0.16b, v5.16b - st1 {v0.2d}, [x6] + mov V5.D[0], x12 + mov V5.D[1], x13 + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V0.16B, V0.16B, V5.16B + st1 {V0.2D}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_256_start_byte @@ -42451,72 +42451,72 @@ L_aes_xts_decrypt_arm64_crypto_256_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v26.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v27.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v28.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v29.16b - eor v0.16b, v0.16b, v30.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V26.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V27.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V28.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V29.16B + eor V0.16B, V0.16B, V30.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_decrypt_arm64_crypto_256_partial_done ENDIF b L_aes_xts_decrypt_arm64_crypto_done ; AES_XTS_128 L_aes_xts_decrypt_arm64_crypto_start_128 IF :LNOT::DEF:NO_AES_128 - ld1 {v24.2d, v25.2d}, [x5], #32 - ld1 {v26.2d}, [x5] - aese v4.16b, v16.16b - aesmc v4.16b, v4.16b - aese v4.16b, v17.16b - aesmc v4.16b, v4.16b - aese v4.16b, v18.16b - aesmc v4.16b, v4.16b - aese v4.16b, v19.16b - aesmc v4.16b, v4.16b - aese v4.16b, v20.16b - aesmc v4.16b, v4.16b - aese v4.16b, v21.16b - aesmc v4.16b, v4.16b - aese v4.16b, v22.16b - aesmc v4.16b, v4.16b - aese v4.16b, v23.16b - aesmc v4.16b, v4.16b - aese v4.16b, v24.16b - aesmc v4.16b, v4.16b - aese v4.16b, v25.16b - eor v4.16b, v4.16b, v26.16b - mov x10, v4.d[0] - mov x11, v4.d[1] - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x4], #0x40 - ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [x4], #0x40 - ld1 {v24.2d, v25.2d}, [x4], #32 - ld1 {v26.2d}, [x4] + ld1 {V24.2D, V25.2D}, [x5], #32 + ld1 {V26.2D}, [x5] + aese V4.16B, V16.16B + aesmc V4.16B, V4.16B + aese V4.16B, V17.16B + aesmc V4.16B, V4.16B + aese V4.16B, V18.16B + aesmc V4.16B, V4.16B + aese V4.16B, V19.16B + aesmc V4.16B, V4.16B + aese V4.16B, V20.16B + aesmc V4.16B, V4.16B + aese V4.16B, V21.16B + aesmc V4.16B, V4.16B + aese V4.16B, V22.16B + aesmc V4.16B, V4.16B + aese V4.16B, V23.16B + aesmc V4.16B, V4.16B + aese V4.16B, V24.16B + aesmc V4.16B, V4.16B + aese V4.16B, V25.16B + eor V4.16B, V4.16B, V26.16B + mov x10, V4.D[0] + mov x11, V4.D[1] + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x4], #0x40 + ld1 {V20.2D, V21.2D, V22.2D, V23.2D}, [x4], #0x40 + ld1 {V24.2D, V25.2D}, [x4], #32 + ld1 {V26.2D}, [x4] and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 @@ -42529,241 +42529,241 @@ L_aes_xts_decrypt_arm64_crypto_start_128 cmp w8, #4 blt L_aes_xts_decrypt_arm64_crypto_128_start_2 L_aes_xts_decrypt_arm64_crypto_128_start_4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - mov v5.d[0], x12 - mov v5.d[1], x13 - mov v6.d[0], x14 - mov v6.d[1], x15 - mov v7.d[0], x16 - mov v7.d[1], x17 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + mov V5.D[0], x12 + mov V5.D[1], x13 + mov V6.D[0], x14 + mov V6.D[1], x15 + mov V7.D[0], x16 + mov V7.D[1], x17 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x17, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v16.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V16.16B + aesimc V2.16B, V2.16B extr x11, x17, x16, #63 - aesd v3.16b, v16.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V16.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B eor x10, x9, x16, lsl 1 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v17.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V17.16B + aesimc V2.16B, V2.16B and x9, x19, x11, asr 63 - aesd v3.16b, v17.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V17.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v18.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V18.16B + aesimc V2.16B, V2.16B eor x12, x9, x10, lsl 1 - aesd v3.16b, v18.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V18.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v19.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V19.16B + aesimc V2.16B, V2.16B extr x15, x13, x12, #63 - aesd v3.16b, v19.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V19.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x14, x9, x12, lsl 1 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v20.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V20.16B + aesimc V2.16B, V2.16B and x9, x19, x15, asr 63 - aesd v3.16b, v20.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V3.16B, V20.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B extr x17, x15, x14, #63 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v21.16b - aesimc v2.16b, v2.16b + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V21.16B + aesimc V2.16B, V2.16B eor x16, x9, x14, lsl 1 - aesd v3.16b, v21.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v22.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v22.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v23.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v23.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v2.16b, v24.16b - aesimc v2.16b, v2.16b - aesd v3.16b, v24.16b - aesimc v3.16b, v3.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - aesd v2.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - aesd v3.16b, v25.16b - eor v3.16b, v3.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V3.16B, V21.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V22.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V22.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V23.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V23.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V2.16B, V24.16B + aesimc V2.16B, V2.16B + aesd V3.16B, V24.16B + aesimc V3.16B, V3.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + aesd V2.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + aesd V3.16B, V25.16B + eor V3.16B, V3.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #4 - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 cmp w8, #4 bge L_aes_xts_decrypt_arm64_crypto_128_start_4 L_aes_xts_decrypt_arm64_crypto_128_start_2 cmp w8, #2 blt L_aes_xts_decrypt_arm64_crypto_128_start_1 - ld1 {v0.16b, v1.16b}, [x0], #32 - mov v5.d[0], x12 - mov v5.d[1], x13 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + mov V5.D[0], x12 + mov V5.D[1], x13 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x13, asr 63 - aesd v1.16b, v16.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V16.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B extr x11, x13, x12, #63 - aesd v1.16b, v17.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V17.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B eor x10, x9, x12, lsl 1 - aesd v1.16b, v18.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V18.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v1.16b, v19.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V19.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B extr x13, x11, x10, #63 - aesd v1.16b, v20.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b + aesd V1.16B, V20.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B eor x12, x9, x10, lsl 1 - aesd v1.16b, v21.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v22.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v23.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v1.16b, v24.16b - aesimc v1.16b, v1.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - aesd v1.16b, v25.16b - eor v1.16b, v1.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - mov v4.d[0], x10 - mov v4.d[1], x11 + aesd V1.16B, V21.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V22.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V23.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V1.16B, V24.16B + aesimc V1.16B, V1.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + aesd V1.16B, V25.16B + eor V1.16B, V1.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + mov V4.D[0], x10 + mov V4.D[1], x11 sub w8, w8, #2 - st1 {v0.16b, v1.16b}, [x1], #32 + st1 {V0.16B, V1.16B}, [x1], #32 L_aes_xts_decrypt_arm64_crypto_128_start_1 cbz w8, L_aes_xts_decrypt_arm64_crypto_128_done - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B and x9, x19, x11, asr 63 - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B extr x11, x11, x10, #63 - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B eor x10, x9, x10, lsl 1 - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - mov v4.d[0], x10 - mov v4.d[1], x11 - st1 {v0.16b}, [x1], #16 + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + mov V4.D[0], x10 + mov V4.D[1], x11 + st1 {V0.16B}, [x1], #16 L_aes_xts_decrypt_arm64_crypto_128_done cbz w2, L_aes_xts_decrypt_arm64_crypto_128_partial_done and x9, x19, x11, asr 63 extr x13, x11, x10, #63 eor x12, x9, x10, lsl 1 - mov v5.d[0], x12 - mov v5.d[1], x13 - ld1 {v0.16b}, [x0], #16 - eor v0.16b, v0.16b, v5.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v0.16b, v0.16b, v5.16b - st1 {v0.2d}, [x6] + mov V5.D[0], x12 + mov V5.D[1], x13 + ld1 {V0.16B}, [x0], #16 + eor V0.16B, V0.16B, V5.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V0.16B, V0.16B, V5.16B + st1 {V0.2D}, [x6] add x1, x1, #16 mov w9, w2 L_aes_xts_decrypt_arm64_crypto_128_start_byte @@ -42776,30 +42776,30 @@ L_aes_xts_decrypt_arm64_crypto_128_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - eor v0.16b, v0.16b, v4.16b - aesd v0.16b, v16.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v17.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v18.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v19.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v20.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v21.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v22.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v23.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v24.16b - aesimc v0.16b, v0.16b - aesd v0.16b, v25.16b - eor v0.16b, v0.16b, v26.16b - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1] + ld1 {V0.2D}, [x6] + eor V0.16B, V0.16B, V4.16B + aesd V0.16B, V16.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V17.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V18.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V19.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V20.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V21.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V22.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V23.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V24.16B + aesimc V0.16B, V0.16B + aesd V0.16B, V25.16B + eor V0.16B, V0.16B, V26.16B + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1] L_aes_xts_decrypt_arm64_crypto_128_partial_done ENDIF L_aes_xts_decrypt_arm64_crypto_done @@ -42812,7 +42812,7 @@ L_aes_xts_decrypt_arm64_crypto_done ENDIF IF :LNOT::DEF:WOLFSSL_ARMASM_NO_NEON IF :DEF:HAVE_AES_DECRYPT :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AESCCM :LOR: :DEF:HAVE_AESGCM :LOR: :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_NEON_te DCB 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5 @@ -42847,7 +42847,7 @@ L_AES_ARM64_NEON_te DCB 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf DCB 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68 DCB 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_NEON_shift_rows_shuffle DCB 0x0c, 0x09, 0x06, 0x03, 0x00, 0x0d, 0x0a, 0x07 @@ -42862,50 +42862,50 @@ AES_invert_key_NEON PROC mov x2, x0 mov w4, w1 L_AES_invert_key_NEON_loop - ld1 {v0.2d}, [x2] - ld1 {v1.2d}, [x3] - st1 {v0.2d}, [x3] - st1 {v1.2d}, [x2], #16 + ld1 {V0.2D}, [x2] + ld1 {V1.2D}, [x3] + st1 {V0.2D}, [x3] + st1 {V1.2D}, [x2], #16 subs w4, w4, #2 sub x3, x3, #16 bne L_AES_invert_key_NEON_loop - movi v2.16b, #27 + movi V2.16B, #27 add x2, x0, #16 sub w4, w1, #1 L_AES_invert_key_NEON_mix_loop - ld1 {v0.2d}, [x2] - sshr v5.16b, v0.16b, #7 - ushr v6.16b, v0.16b, #6 - ushr v3.16b, v0.16b, #5 - and v5.16b, v5.16b, v2.16b - pmul v6.16b, v6.16b, v2.16b - pmul v3.16b, v3.16b, v2.16b - shl v4.16b, v0.16b, #1 - eor v5.16b, v5.16b, v4.16b - shl v4.16b, v0.16b, #3 - eor v3.16b, v3.16b, v4.16b - shl v4.16b, v0.16b, #2 - eor v6.16b, v6.16b, v4.16b - eor v4.16b, v5.16b, v3.16b - eor v3.16b, v3.16b, v0.16b - eor v5.16b, v6.16b, v3.16b - eor v6.16b, v6.16b, v4.16b - eor v4.16b, v4.16b, v0.16b - shl v0.4s, v4.4s, #8 - rev32 v5.8h, v5.8h - sri v0.4s, v4.4s, #24 - eor v0.16b, v0.16b, v6.16b - shl v4.4s, v3.4s, #24 - eor v0.16b, v0.16b, v5.16b - sri v4.4s, v3.4s, #8 - eor v0.16b, v0.16b, v4.16b - st1 {v0.2d}, [x2], #16 + ld1 {V0.2D}, [x2] + sshr V5.16B, V0.16B, #7 + ushr V6.16B, V0.16B, #6 + ushr V3.16B, V0.16B, #5 + and V5.16B, V5.16B, V2.16B + pmul V6.16B, V6.16B, V2.16B + pmul V3.16B, V3.16B, V2.16B + shl V4.16B, V0.16B, #1 + eor V5.16B, V5.16B, V4.16B + shl V4.16B, V0.16B, #3 + eor V3.16B, V3.16B, V4.16B + shl V4.16B, V0.16B, #2 + eor V6.16B, V6.16B, V4.16B + eor V4.16B, V5.16B, V3.16B + eor V3.16B, V3.16B, V0.16B + eor V5.16B, V6.16B, V3.16B + eor V6.16B, V6.16B, V4.16B + eor V4.16B, V4.16B, V0.16B + shl V0.4S, V4.4S, #8 + rev32 V5.8H, V5.8H + sri V0.4S, V4.4S, #24 + eor V0.16B, V0.16B, V6.16B + shl V4.4S, V3.4S, #24 + eor V0.16B, V0.16B, V5.16B + sri V4.4S, V3.4S, #8 + eor V0.16B, V0.16B, V4.16B + st1 {V0.2D}, [x2], #16 subs w4, w4, #1 bne L_AES_invert_key_NEON_mix_loop ret ENDP ENDIF - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_NEON_rcon DCD 0x01000000, 0x02000000, 0x04000000, 0x08000000 @@ -42917,223 +42917,223 @@ L_AES_ARM64_NEON_rcon AES_set_encrypt_key_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x4, L_AES_ARM64_NEON_rcon add x4, x4, L_AES_ARM64_NEON_rcon adrp x5, L_AES_ARM64_NEON_te add x5, x5, L_AES_ARM64_NEON_te - ld1 {v6.16b, v7.16b, v8.16b, v9.16b}, [x5], #0x40 - ld1 {v10.16b, v11.16b, v12.16b, v13.16b}, [x5], #0x40 - ld1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x5], #0x40 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x5] - movi v2.16b, #0x40 - movi v3.16b, #0x80 - movi v4.16b, #0xc0 - movi v5.16b, #27 - eor v26.16b, v26.16b, v26.16b + ld1 {V6.16B, V7.16B, V8.16B, V9.16B}, [x5], #0x40 + ld1 {V10.16B, V11.16B, V12.16B, V13.16B}, [x5], #0x40 + ld1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x5], #0x40 + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x5] + movi V2.16B, #0x40 + movi V3.16B, #0x80 + movi V4.16B, #0xc0 + movi V5.16B, #27 + eor V26.16B, V26.16B, V26.16B cmp x1, #0x80 beq L_AES_set_encrypt_key_NEON_start_128 cmp x1, #0xc0 beq L_AES_set_encrypt_key_NEON_start_192 - ld1 {v0.16b}, [x0], #16 - ld1 {v1.16b}, [x0] - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - st1 {v0.2d}, [x2], #16 - st1 {v1.2d}, [x2], #16 + ld1 {V0.16B}, [x0], #16 + ld1 {V1.16B}, [x0] + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + st1 {V0.2D}, [x2], #16 + st1 {V1.2D}, [x2], #16 mov x3, #6 L_AES_set_encrypt_key_NEON_loop_256 - eor v22.16b, v1.16b, v2.16b - eor v23.16b, v1.16b, v3.16b - eor v24.16b, v1.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - shl v22.4s, v25.4s, #8 - sri v22.4s, v25.4s, #24 - eor v0.16b, v0.16b, v22.16b - ld1r {v25.4s}, [x4], #4 - dup v22.4s, v0.s[0] - dup v23.2s, v0.s[1] - dup v24.2s, v0.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v0.16b, v0.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v0.16b, v0.16b, v23.16b - eor v0.16b, v0.16b, v24.16b - eor v0.16b, v0.16b, v25.16b - st1 {v0.2d}, [x2], #16 - eor v22.16b, v0.16b, v2.16b - eor v23.16b, v0.16b, v3.16b - eor v24.16b, v0.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v0.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - eor v1.16b, v1.16b, v25.16b - dup v22.4s, v1.s[0] - dup v23.2s, v1.s[1] - dup v24.2s, v1.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v1.16b, v1.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v1.16b, v1.16b, v23.16b - eor v1.16b, v1.16b, v24.16b - st1 {v1.2d}, [x2], #16 + eor V22.16B, V1.16B, V2.16B + eor V23.16B, V1.16B, V3.16B + eor V24.16B, V1.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + shl V22.4S, V25.4S, #8 + sri V22.4S, V25.4S, #24 + eor V0.16B, V0.16B, V22.16B + ld1r {V25.4S}, [x4], #4 + dup V22.4S, V0.S[0] + dup V23.2S, V0.S[1] + dup V24.2S, V0.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V0.16B, V0.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V0.16B, V0.16B, V23.16B + eor V0.16B, V0.16B, V24.16B + eor V0.16B, V0.16B, V25.16B + st1 {V0.2D}, [x2], #16 + eor V22.16B, V0.16B, V2.16B + eor V23.16B, V0.16B, V3.16B + eor V24.16B, V0.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V0.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + eor V1.16B, V1.16B, V25.16B + dup V22.4S, V1.S[0] + dup V23.2S, V1.S[1] + dup V24.2S, V1.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V1.16B, V1.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V1.16B, V1.16B, V23.16B + eor V1.16B, V1.16B, V24.16B + st1 {V1.2D}, [x2], #16 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_256 - eor v22.16b, v1.16b, v2.16b - eor v23.16b, v1.16b, v3.16b - eor v24.16b, v1.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - shl v22.4s, v25.4s, #8 - sri v22.4s, v25.4s, #24 - eor v0.16b, v0.16b, v22.16b - ld1r {v25.4s}, [x4], #4 - dup v22.4s, v0.s[0] - dup v23.2s, v0.s[1] - dup v24.2s, v0.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v0.16b, v0.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v0.16b, v0.16b, v23.16b - eor v0.16b, v0.16b, v24.16b - eor v0.16b, v0.16b, v25.16b - st1 {v0.2d}, [x2], #16 + eor V22.16B, V1.16B, V2.16B + eor V23.16B, V1.16B, V3.16B + eor V24.16B, V1.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + shl V22.4S, V25.4S, #8 + sri V22.4S, V25.4S, #24 + eor V0.16B, V0.16B, V22.16B + ld1r {V25.4S}, [x4], #4 + dup V22.4S, V0.S[0] + dup V23.2S, V0.S[1] + dup V24.2S, V0.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V0.16B, V0.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V0.16B, V0.16B, V23.16B + eor V0.16B, V0.16B, V24.16B + eor V0.16B, V0.16B, V25.16B + st1 {V0.2D}, [x2], #16 b L_AES_set_encrypt_key_NEON_end L_AES_set_encrypt_key_NEON_start_192 - ld1 {v0.16b}, [x0], #16 - ld1 {v1.8b}, [x0] - rev32 v0.16b, v0.16b - rev32 v1.8b, v1.8b - st1 {v0.16b}, [x2], #16 - st1 {v1.8b}, [x2], #8 - ext v1.16b, v1.16b, v1.16b, #8 + ld1 {V0.16B}, [x0], #16 + ld1 {V1.8B}, [x0] + rev32 V0.16B, V0.16B + rev32 V1.8B, V1.8B + st1 {V0.16B}, [x2], #16 + st1 {V1.8B}, [x2], #8 + ext8 V1.16B, V1.16B, V1.16B, #8 mov x3, #7 L_AES_set_encrypt_key_NEON_loop_192 - eor v22.16b, v1.16b, v2.16b - eor v23.16b, v1.16b, v3.16b - eor v24.16b, v1.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - shl v22.4s, v25.4s, #8 - sri v22.4s, v25.4s, #24 - eor v0.16b, v0.16b, v22.16b - ld1r {v25.4s}, [x4], #4 - dup v22.4s, v0.s[0] - dup v23.2s, v0.s[1] - dup v24.2s, v0.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v0.16b, v0.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v0.16b, v0.16b, v23.16b - eor v0.16b, v0.16b, v24.16b - eor v0.16b, v0.16b, v25.16b - st1 {v0.2d}, [x2], #16 - mov v23.16b, v26.16b - mov v23.s[2], v0.s[3] - eor v1.16b, v1.16b, v23.16b - mov v23.16b, v26.16b - mov v23.s[3], v1.s[2] - eor v1.16b, v1.16b, v23.16b - st1 {v1.d}[1], [x2], #8 + eor V22.16B, V1.16B, V2.16B + eor V23.16B, V1.16B, V3.16B + eor V24.16B, V1.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + shl V22.4S, V25.4S, #8 + sri V22.4S, V25.4S, #24 + eor V0.16B, V0.16B, V22.16B + ld1r {V25.4S}, [x4], #4 + dup V22.4S, V0.S[0] + dup V23.2S, V0.S[1] + dup V24.2S, V0.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V0.16B, V0.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V0.16B, V0.16B, V23.16B + eor V0.16B, V0.16B, V24.16B + eor V0.16B, V0.16B, V25.16B + st1 {V0.2D}, [x2], #16 + mov V23.16B, V26.16B + mov V23.S[2], V0.S[3] + eor V1.16B, V1.16B, V23.16B + mov V23.16B, V26.16B + mov V23.S[3], V1.S[2] + eor V1.16B, V1.16B, V23.16B + st1 {V1.D}[1], [x2], #8 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_192 - eor v22.16b, v1.16b, v2.16b - eor v23.16b, v1.16b, v3.16b - eor v24.16b, v1.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v1.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - shl v22.4s, v25.4s, #8 - sri v22.4s, v25.4s, #24 - eor v0.16b, v0.16b, v22.16b - ld1r {v25.4s}, [x4], #4 - dup v22.4s, v0.s[0] - dup v23.2s, v0.s[1] - dup v24.2s, v0.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v0.16b, v0.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v0.16b, v0.16b, v23.16b - eor v0.16b, v0.16b, v24.16b - eor v0.16b, v0.16b, v25.16b - st1 {v0.2d}, [x2], #16 + eor V22.16B, V1.16B, V2.16B + eor V23.16B, V1.16B, V3.16B + eor V24.16B, V1.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V1.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + shl V22.4S, V25.4S, #8 + sri V22.4S, V25.4S, #24 + eor V0.16B, V0.16B, V22.16B + ld1r {V25.4S}, [x4], #4 + dup V22.4S, V0.S[0] + dup V23.2S, V0.S[1] + dup V24.2S, V0.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V0.16B, V0.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V0.16B, V0.16B, V23.16B + eor V0.16B, V0.16B, V24.16B + eor V0.16B, V0.16B, V25.16B + st1 {V0.2D}, [x2], #16 b L_AES_set_encrypt_key_NEON_end L_AES_set_encrypt_key_NEON_start_128 - ld1 {v0.16b}, [x0] - rev32 v0.16b, v0.16b - st1 {v0.2d}, [x2], #16 + ld1 {V0.16B}, [x0] + rev32 V0.16B, V0.16B + st1 {V0.2D}, [x2], #16 mov x3, #10 L_AES_set_encrypt_key_NEON_loop_128 - eor v22.16b, v0.16b, v2.16b - eor v23.16b, v0.16b, v3.16b - eor v24.16b, v0.16b, v4.16b - tbl v25.16b, {v6.16b, v7.16b, v8.16b, v9.16b}, v0.16b - tbl v22.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v22.16b - tbl v23.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v23.16b - tbl v24.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v24.16b - orr v25.16b, v25.16b, v22.16b - orr v23.16b, v23.16b, v24.16b - orr v25.16b, v25.16b, v23.16b - ext v25.16b, v25.16b, v26.16b, #12 - shl v22.4s, v25.4s, #8 - sri v22.4s, v25.4s, #24 - eor v0.16b, v0.16b, v22.16b - ld1r {v25.4s}, [x4], #4 - dup v22.4s, v0.s[0] - dup v23.2s, v0.s[1] - dup v24.2s, v0.s[2] - ext v22.16b, v26.16b, v22.16b, #12 - ext v23.16b, v26.16b, v23.16b, #8 - eor v0.16b, v0.16b, v22.16b - ext v24.16b, v26.16b, v24.16b, #4 - eor v0.16b, v0.16b, v23.16b - eor v0.16b, v0.16b, v24.16b - eor v0.16b, v0.16b, v25.16b - st1 {v0.2d}, [x2], #16 + eor V22.16B, V0.16B, V2.16B + eor V23.16B, V0.16B, V3.16B + eor V24.16B, V0.16B, V4.16B + tbl V25.16B, {V6.16B, V7.16B, V8.16B, V9.16B}, V0.16B + tbl V22.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V22.16B + tbl V23.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V23.16B + tbl V24.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V24.16B + orr V25.16B, V25.16B, V22.16B + orr V23.16B, V23.16B, V24.16B + orr V25.16B, V25.16B, V23.16B + ext8 V25.16B, V25.16B, V26.16B, #12 + shl V22.4S, V25.4S, #8 + sri V22.4S, V25.4S, #24 + eor V0.16B, V0.16B, V22.16B + ld1r {V25.4S}, [x4], #4 + dup V22.4S, V0.S[0] + dup V23.2S, V0.S[1] + dup V24.2S, V0.S[2] + ext8 V22.16B, V26.16B, V22.16B, #12 + ext8 V23.16B, V26.16B, V23.16B, #8 + eor V0.16B, V0.16B, V22.16B + ext8 V24.16B, V26.16B, V24.16B, #4 + eor V0.16B, V0.16B, V23.16B + eor V0.16B, V0.16B, V24.16B + eor V0.16B, V0.16B, V25.16B + st1 {V0.2D}, [x2], #16 subs x3, x3, #1 bne L_AES_set_encrypt_key_NEON_loop_128 L_AES_set_encrypt_key_NEON_end - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -43144,755 +43144,755 @@ L_AES_set_encrypt_key_NEON_end AES_ECB_encrypt_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x5, L_AES_ARM64_NEON_te add x5, x5, L_AES_ARM64_NEON_te adrp x6, L_AES_ARM64_NEON_shift_rows_shuffle add x6, x6, L_AES_ARM64_NEON_shift_rows_shuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x5], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x5], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x5], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x5], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x5], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x5], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x5] cmp x2, #0x40 blt L_AES_ECB_encrypt_NEON_start_2 L_AES_ECB_encrypt_NEON_loop_4 mov x8, x3 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - movi v4.16b, #27 - and v8.16b, v8.16b, v4.16b - and v9.16b, v9.16b, v4.16b - and v10.16b, v10.16b, v4.16b - and v11.16b, v11.16b, v4.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - eor v6.16b, v10.16b, v2.16b - eor v7.16b, v11.16b, v3.16b - shl v12.4s, v4.4s, #8 - shl v13.4s, v5.4s, #8 - shl v14.4s, v6.4s, #8 - shl v15.4s, v7.4s, #8 - sri v12.4s, v4.4s, #24 - sri v13.4s, v5.4s, #24 - sri v14.4s, v6.4s, #24 - sri v15.4s, v7.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - shl v6.4s, v2.4s, #24 - shl v7.4s, v3.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - sri v6.4s, v2.4s, #8 - sri v7.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + movi V4.16B, #27 + and V8.16B, V8.16B, V4.16B + and V9.16B, V9.16B, V4.16B + and V10.16B, V10.16B, V4.16B + and V11.16B, V11.16B, V4.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + eor V6.16B, V10.16B, V2.16B + eor V7.16B, V11.16B, V3.16B + shl V12.4S, V4.4S, #8 + shl V13.4S, V5.4S, #8 + shl V14.4S, V6.4S, #8 + shl V15.4S, V7.4S, #8 + sri V12.4S, V4.4S, #24 + sri V13.4S, V5.4S, #24 + sri V14.4S, V6.4S, #24 + sri V15.4S, V7.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + shl V6.4S, V2.4S, #24 + shl V7.4S, V3.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + sri V6.4S, V2.4S, #8 + sri V7.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B ; Round Done subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_ECB_encrypt_NEON_loop_4 L_AES_ECB_encrypt_NEON_start_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 cmp x2, #16 beq L_AES_ECB_encrypt_NEON_start_1 blt L_AES_ECB_encrypt_NEON_data_done L_AES_ECB_encrypt_NEON_loop_2 mov x8, x3 - ld1 {v0.16b, v1.16b}, [x0], #32 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v10.16b, v0.16b, #1 - shl v11.16b, v1.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - shl v10.4s, v4.4s, #8 - shl v11.4s, v5.4s, #8 - sri v10.4s, v4.4s, #24 - sri v11.4s, v5.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V10.16B, V0.16B, #1 + shl V11.16B, V1.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + shl V10.4S, V4.4S, #8 + shl V11.4S, V5.4S, #8 + sri V10.4S, V4.4S, #24 + sri V11.4S, V5.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; Round Done subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + st1 {V0.16B, V1.16B}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_ECB_encrypt_NEON_data_done L_AES_ECB_encrypt_NEON_start_1 - ld1 {v3.2d}, [x6] + ld1 {V3.2D}, [x6] mov x8, x3 - ld1 {v0.16b}, [x0], #16 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B sub w7, w4, #2 L_AES_ECB_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x8], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x8], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x8], #16 - sshr v10.16b, v0.16b, #7 - shl v9.16b, v0.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v0.8h - eor v11.16b, v10.16b, v0.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v0.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x8], #16 + sshr V10.16B, V0.16B, #7 + shl V9.16B, V0.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V0.8H + eor V11.16B, V10.16B, V0.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V0.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v0.4s, #8 - sri v8.4s, v11.4s, #24 - eor v0.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V0.4S, #8 + sri V8.4S, V11.4S, #24 + eor V0.16B, V10.16B, V9.16B + eor V0.16B, V0.16B, V8.16B subs w7, w7, #2 bne L_AES_ECB_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x8], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x8], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x8], #16 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x8], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + st1 {V0.16B}, [x1], #16 L_AES_ECB_encrypt_NEON_data_done - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -43904,140 +43904,140 @@ L_AES_ECB_encrypt_NEON_data_done AES_CBC_encrypt_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x6, L_AES_ARM64_NEON_te add x6, x6, L_AES_ARM64_NEON_te adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle add x7, x7, L_AES_ARM64_NEON_shift_rows_shuffle - ld1 {v10.16b, v11.16b, v12.16b, v13.16b}, [x6], #0x40 - ld1 {v14.16b, v15.16b, v16.16b, v17.16b}, [x6], #0x40 - ld1 {v18.16b, v19.16b, v20.16b, v21.16b}, [x6], #0x40 - ld1 {v22.16b, v23.16b, v24.16b, v25.16b}, [x6] - movi v6.16b, #0x40 - movi v7.16b, #0x80 - movi v8.16b, #0xc0 - movi v9.16b, #27 - ld1 {v0.2d}, [x5] - ld1 {v26.2d}, [x7] + ld1 {V10.16B, V11.16B, V12.16B, V13.16B}, [x6], #0x40 + ld1 {V14.16B, V15.16B, V16.16B, V17.16B}, [x6], #0x40 + ld1 {V18.16B, V19.16B, V20.16B, V21.16B}, [x6], #0x40 + ld1 {V22.16B, V23.16B, V24.16B, V25.16B}, [x6] + movi V6.16B, #0x40 + movi V7.16B, #0x80 + movi V8.16B, #0xc0 + movi V9.16B, #27 + ld1 {V0.2D}, [x5] + ld1 {V26.2D}, [x7] L_AES_CBC_encrypt_NEON_loop_block add x9, x3, #16 - ld1 {v1.16b}, [x0], #16 - ld1 {v2.16b}, [x3] - eor v0.16b, v0.16b, v1.16b - rev32 v0.16b, v0.16b + ld1 {V1.16B}, [x0], #16 + ld1 {V2.16B}, [x3] + eor V0.16B, V0.16B, V1.16B + rev32 V0.16B, V0.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v2.16b + eor V0.16B, V0.16B, V2.16B sub w8, w4, #2 L_AES_CBC_encrypt_NEON_loop_nr - eor v2.16b, v0.16b, v6.16b - eor v3.16b, v0.16b, v7.16b - eor v4.16b, v0.16b, v8.16b - tbl v1.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v0.16b - tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b - tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b - tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b - orr v1.16b, v1.16b, v2.16b - orr v3.16b, v3.16b, v4.16b - orr v1.16b, v1.16b, v3.16b - tbl v1.16b, {v1.16b}, v26.16b - ld1 {v0.2d}, [x9], #16 - sshr v4.16b, v1.16b, #7 - shl v3.16b, v1.16b, #1 - and v4.16b, v4.16b, v9.16b - eor v4.16b, v4.16b, v3.16b - rev32 v2.8h, v1.8h - eor v5.16b, v4.16b, v1.16b - eor v4.16b, v4.16b, v2.16b - shl v3.4s, v1.4s, #24 - shl v2.4s, v5.4s, #8 + eor V2.16B, V0.16B, V6.16B + eor V3.16B, V0.16B, V7.16B + eor V4.16B, V0.16B, V8.16B + tbl V1.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V0.16B + tbl V2.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V2.16B + tbl V3.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V3.16B + tbl V4.16B, {V22.16B, V23.16B, V24.16B, V25.16B}, V4.16B + orr V1.16B, V1.16B, V2.16B + orr V3.16B, V3.16B, V4.16B + orr V1.16B, V1.16B, V3.16B + tbl V1.16B, {V1.16B}, V26.16B + ld1 {V0.2D}, [x9], #16 + sshr V4.16B, V1.16B, #7 + shl V3.16B, V1.16B, #1 + and V4.16B, V4.16B, V9.16B + eor V4.16B, V4.16B, V3.16B + rev32 V2.8H, V1.8H + eor V5.16B, V4.16B, V1.16B + eor V4.16B, V4.16B, V2.16B + shl V3.4S, V1.4S, #24 + shl V2.4S, V5.4S, #8 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - sri v3.4s, v1.4s, #8 - sri v2.4s, v5.4s, #24 - eor v1.16b, v4.16b, v3.16b - eor v1.16b, v1.16b, v2.16b - eor v2.16b, v1.16b, v6.16b - eor v3.16b, v1.16b, v7.16b - eor v4.16b, v1.16b, v8.16b - tbl v0.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v1.16b - tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b - tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b - tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b - orr v0.16b, v0.16b, v2.16b - orr v3.16b, v3.16b, v4.16b - orr v0.16b, v0.16b, v3.16b - tbl v0.16b, {v0.16b}, v26.16b - ld1 {v1.2d}, [x9], #16 - sshr v4.16b, v0.16b, #7 - shl v3.16b, v0.16b, #1 - and v4.16b, v4.16b, v9.16b - eor v4.16b, v4.16b, v3.16b - rev32 v2.8h, v0.8h - eor v5.16b, v4.16b, v0.16b - eor v4.16b, v4.16b, v2.16b - shl v3.4s, v0.4s, #24 - shl v2.4s, v5.4s, #8 + eor V4.16B, V4.16B, V0.16B + sri V3.4S, V1.4S, #8 + sri V2.4S, V5.4S, #24 + eor V1.16B, V4.16B, V3.16B + eor V1.16B, V1.16B, V2.16B + eor V2.16B, V1.16B, V6.16B + eor V3.16B, V1.16B, V7.16B + eor V4.16B, V1.16B, V8.16B + tbl V0.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V1.16B + tbl V2.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V2.16B + tbl V3.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V3.16B + tbl V4.16B, {V22.16B, V23.16B, V24.16B, V25.16B}, V4.16B + orr V0.16B, V0.16B, V2.16B + orr V3.16B, V3.16B, V4.16B + orr V0.16B, V0.16B, V3.16B + tbl V0.16B, {V0.16B}, V26.16B + ld1 {V1.2D}, [x9], #16 + sshr V4.16B, V0.16B, #7 + shl V3.16B, V0.16B, #1 + and V4.16B, V4.16B, V9.16B + eor V4.16B, V4.16B, V3.16B + rev32 V2.8H, V0.8H + eor V5.16B, V4.16B, V0.16B + eor V4.16B, V4.16B, V2.16B + shl V3.4S, V0.4S, #24 + shl V2.4S, V5.4S, #8 ; XOR in Key Schedule - eor v4.16b, v4.16b, v1.16b - sri v3.4s, v0.4s, #8 - sri v2.4s, v5.4s, #24 - eor v0.16b, v4.16b, v3.16b - eor v0.16b, v0.16b, v2.16b + eor V4.16B, V4.16B, V1.16B + sri V3.4S, V0.4S, #8 + sri V2.4S, V5.4S, #24 + eor V0.16B, V4.16B, V3.16B + eor V0.16B, V0.16B, V2.16B subs w8, w8, #2 bne L_AES_CBC_encrypt_NEON_loop_nr - eor v2.16b, v0.16b, v6.16b - eor v3.16b, v0.16b, v7.16b - eor v4.16b, v0.16b, v8.16b - tbl v1.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v0.16b - tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b - tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b - tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b - orr v1.16b, v1.16b, v2.16b - orr v3.16b, v3.16b, v4.16b - orr v1.16b, v1.16b, v3.16b - tbl v1.16b, {v1.16b}, v26.16b - ld1 {v0.2d}, [x9], #16 - sshr v4.16b, v1.16b, #7 - shl v3.16b, v1.16b, #1 - and v4.16b, v4.16b, v9.16b - eor v4.16b, v4.16b, v3.16b - rev32 v2.8h, v1.8h - eor v5.16b, v4.16b, v1.16b - eor v4.16b, v4.16b, v2.16b - shl v3.4s, v1.4s, #24 - shl v2.4s, v5.4s, #8 + eor V2.16B, V0.16B, V6.16B + eor V3.16B, V0.16B, V7.16B + eor V4.16B, V0.16B, V8.16B + tbl V1.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V0.16B + tbl V2.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V2.16B + tbl V3.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V3.16B + tbl V4.16B, {V22.16B, V23.16B, V24.16B, V25.16B}, V4.16B + orr V1.16B, V1.16B, V2.16B + orr V3.16B, V3.16B, V4.16B + orr V1.16B, V1.16B, V3.16B + tbl V1.16B, {V1.16B}, V26.16B + ld1 {V0.2D}, [x9], #16 + sshr V4.16B, V1.16B, #7 + shl V3.16B, V1.16B, #1 + and V4.16B, V4.16B, V9.16B + eor V4.16B, V4.16B, V3.16B + rev32 V2.8H, V1.8H + eor V5.16B, V4.16B, V1.16B + eor V4.16B, V4.16B, V2.16B + shl V3.4S, V1.4S, #24 + shl V2.4S, V5.4S, #8 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - sri v3.4s, v1.4s, #8 - sri v2.4s, v5.4s, #24 - eor v1.16b, v4.16b, v3.16b - eor v1.16b, v1.16b, v2.16b - eor v2.16b, v1.16b, v6.16b - eor v3.16b, v1.16b, v7.16b - eor v4.16b, v1.16b, v8.16b - tbl v0.16b, {v10.16b, v11.16b, v12.16b, v13.16b}, v1.16b - tbl v2.16b, {v14.16b, v15.16b, v16.16b, v17.16b}, v2.16b - tbl v3.16b, {v18.16b, v19.16b, v20.16b, v21.16b}, v3.16b - tbl v4.16b, {v22.16b, v23.16b, v24.16b, v25.16b}, v4.16b - orr v0.16b, v0.16b, v2.16b - orr v3.16b, v3.16b, v4.16b - orr v0.16b, v0.16b, v3.16b - tbl v0.16b, {v0.16b}, v26.16b - ld1 {v1.2d}, [x9], #16 + eor V4.16B, V4.16B, V0.16B + sri V3.4S, V1.4S, #8 + sri V2.4S, V5.4S, #24 + eor V1.16B, V4.16B, V3.16B + eor V1.16B, V1.16B, V2.16B + eor V2.16B, V1.16B, V6.16B + eor V3.16B, V1.16B, V7.16B + eor V4.16B, V1.16B, V8.16B + tbl V0.16B, {V10.16B, V11.16B, V12.16B, V13.16B}, V1.16B + tbl V2.16B, {V14.16B, V15.16B, V16.16B, V17.16B}, V2.16B + tbl V3.16B, {V18.16B, V19.16B, V20.16B, V21.16B}, V3.16B + tbl V4.16B, {V22.16B, V23.16B, V24.16B, V25.16B}, V4.16B + orr V0.16B, V0.16B, V2.16B + orr V3.16B, V3.16B, V4.16B + orr V0.16B, V0.16B, V3.16B + tbl V0.16B, {V0.16B}, V26.16B + ld1 {V1.2D}, [x9], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v1.16b - rev32 v0.16b, v0.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V1.16B + rev32 V0.16B, V0.16B + st1 {V0.16B}, [x1], #16 subs x2, x2, #16 bne L_AES_CBC_encrypt_NEON_loop_block - st1 {v0.2d}, [x5] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st1 {V0.2D}, [x5] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -44049,810 +44049,810 @@ L_AES_CBC_encrypt_NEON_loop_nr AES_CTR_encrypt_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x6, L_AES_ARM64_NEON_te add x6, x6, L_AES_ARM64_NEON_te adrp x7, L_AES_ARM64_NEON_shift_rows_shuffle add x7, x7, L_AES_ARM64_NEON_shift_rows_shuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x6], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x6], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x6], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] - ld1 {v2.2d}, [x5] - rev64 v8.16b, v2.16b - rev32 v2.16b, v2.16b - mov x10, v8.d[1] - mov x11, v8.d[0] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x6], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x6], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x6], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x6] + ld1 {V2.2D}, [x5] + rev64 V8.16B, V2.16B + rev32 V2.16B, V2.16B + mov x10, V8.D[1] + mov x11, V8.D[0] cmp x2, #0x40 blt L_AES_CTR_encrypt_NEON_start_2 L_AES_CTR_encrypt_NEON_loop_4 mov x9, x3 - ld1 {v4.2d}, [x9], #16 - mov v8.d[1], x10 - mov v8.d[0], x11 - rev64 v8.4s, v8.4s + ld1 {V4.2D}, [x9], #16 + mov V8.D[1], x10 + mov V8.D[0], x11 + rev64 V8.4S, V8.4S ; Round: 0 - XOR in key schedule - eor v0.16b, v8.16b, v4.16b + eor V0.16B, V8.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v8.d[1], x10 - mov v8.d[0], x11 - rev64 v8.4s, v8.4s - eor v1.16b, v8.16b, v4.16b + mov V8.D[1], x10 + mov V8.D[0], x11 + rev64 V8.4S, V8.4S + eor V1.16B, V8.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v8.d[1], x10 - mov v8.d[0], x11 - rev64 v8.4s, v8.4s - eor v2.16b, v8.16b, v4.16b + mov V8.D[1], x10 + mov V8.D[0], x11 + rev64 V8.4S, V8.4S + eor V2.16B, V8.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v8.d[1], x10 - mov v8.d[0], x11 - rev64 v8.4s, v8.4s - eor v3.16b, v8.16b, v4.16b + mov V8.D[1], x10 + mov V8.D[0], x11 + rev64 V8.4S, V8.4S + eor V3.16B, V8.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v8.d[1], x10 - mov v8.d[0], x11 - rev64 v8.4s, v8.4s + mov V8.D[1], x10 + mov V8.D[0], x11 + rev64 V8.4S, V8.4S sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x7] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x7] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x7] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - movi v4.16b, #27 - and v8.16b, v8.16b, v4.16b - and v9.16b, v9.16b, v4.16b - and v10.16b, v10.16b, v4.16b - and v11.16b, v11.16b, v4.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - eor v6.16b, v10.16b, v2.16b - eor v7.16b, v11.16b, v3.16b - shl v12.4s, v4.4s, #8 - shl v13.4s, v5.4s, #8 - shl v14.4s, v6.4s, #8 - shl v15.4s, v7.4s, #8 - sri v12.4s, v4.4s, #24 - sri v13.4s, v5.4s, #24 - sri v14.4s, v6.4s, #24 - sri v15.4s, v7.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - shl v6.4s, v2.4s, #24 - shl v7.4s, v3.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - sri v6.4s, v2.4s, #8 - sri v7.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x7] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + movi V4.16B, #27 + and V8.16B, V8.16B, V4.16B + and V9.16B, V9.16B, V4.16B + and V10.16B, V10.16B, V4.16B + and V11.16B, V11.16B, V4.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + eor V6.16B, V10.16B, V2.16B + eor V7.16B, V11.16B, V3.16B + shl V12.4S, V4.4S, #8 + shl V13.4S, V5.4S, #8 + shl V14.4S, V6.4S, #8 + shl V15.4S, V7.4S, #8 + sri V12.4S, V4.4S, #24 + sri V13.4S, V5.4S, #24 + sri V14.4S, V6.4S, #24 + sri V15.4S, V7.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + shl V6.4S, V2.4S, #24 + shl V7.4S, V3.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + sri V6.4S, V2.4S, #8 + sri V7.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b + ld1 {V4.2D}, [x9], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B ; Round Done subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x7] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x7] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x7] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x7] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_CTR_encrypt_NEON_loop_4 - mov v2.d[1], x10 - mov v2.d[0], x11 - rev64 v2.4s, v2.4s + mov V2.D[1], x10 + mov V2.D[0], x11 + rev64 V2.4S, V2.4S L_AES_CTR_encrypt_NEON_start_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 cmp x2, #16 beq L_AES_CTR_encrypt_NEON_start_1 blt L_AES_CTR_encrypt_NEON_data_done L_AES_CTR_encrypt_NEON_loop_2 mov x9, x3 - ld1 {v4.2d}, [x9], #16 + ld1 {V4.2D}, [x9], #16 ; Round: 0 - XOR in key schedule - eor v0.16b, v2.16b, v4.16b + eor V0.16B, V2.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v2.d[1], x10 - mov v2.d[0], x11 - rev64 v2.4s, v2.4s - eor v1.16b, v2.16b, v4.16b + mov V2.D[1], x10 + mov V2.D[0], x11 + rev64 V2.4S, V2.4S + eor V1.16B, V2.16B, V4.16B adds x10, x10, #1 adc x11, x11, xzr - mov v2.d[1], x10 - mov v2.d[0], x11 - rev64 v2.4s, v2.4s + mov V2.D[1], x10 + mov V2.D[0], x11 + rev64 V2.4S, V2.4S sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x7] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x7] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x7] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v10.16b, v0.16b, #1 - shl v11.16b, v1.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - shl v10.4s, v4.4s, #8 - shl v11.4s, v5.4s, #8 - sri v10.4s, v4.4s, #24 - sri v11.4s, v5.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x7] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V10.16B, V0.16B, #1 + shl V11.16B, V1.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + shl V10.4S, V4.4S, #8 + shl V11.4S, V5.4S, #8 + sri V10.4S, V4.4S, #24 + sri V11.4S, V5.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + ld1 {V4.2D}, [x9], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; Round Done subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x7] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x7] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x7] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x7] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - ld1 {v4.16b, v5.16b}, [x0], #32 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + ld1 {V4.16B, V5.16B}, [x0], #32 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + st1 {V0.16B, V1.16B}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_CTR_encrypt_NEON_data_done L_AES_CTR_encrypt_NEON_start_1 - ld1 {v3.2d}, [x7] + ld1 {V3.2D}, [x7] mov x9, x3 - ld1 {v4.2d}, [x9], #16 + ld1 {V4.2D}, [x9], #16 ; Round: 0 - XOR in key schedule - eor v0.16b, v2.16b, v4.16b + eor V0.16B, V2.16B, V4.16B sub w8, w4, #2 L_AES_CTR_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x9], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x9], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x9], #16 - sshr v10.16b, v0.16b, #7 - shl v9.16b, v0.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v0.8h - eor v11.16b, v10.16b, v0.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v0.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x9], #16 + sshr V10.16B, V0.16B, #7 + shl V9.16B, V0.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V0.8H + eor V11.16B, V10.16B, V0.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V0.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v0.4s, #8 - sri v8.4s, v11.4s, #24 - eor v0.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V0.4S, #8 + sri V8.4S, V11.4S, #24 + eor V0.16B, V10.16B, V9.16B + eor V0.16B, V0.16B, V8.16B subs w8, w8, #2 bne L_AES_CTR_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x9], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x9], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x9], #16 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x9], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - ld1 {v4.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + ld1 {V4.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1], #16 adds x10, x10, #1 adc x11, x11, xzr - mov v2.d[1], x10 - mov v2.d[0], x11 - rev64 v2.4s, v2.4s + mov V2.D[1], x10 + mov V2.D[0], x11 + rev64 V2.4S, V2.4S L_AES_CTR_encrypt_NEON_data_done - rev32 v2.16b, v2.16b - st1 {v2.2d}, [x5] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + rev32 V2.16B, V2.16B + st1 {V2.2D}, [x5] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP ENDIF IF :DEF:HAVE_AES_DECRYPT IF :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AES_ECB - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_NEON_td DCB 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 @@ -44887,7 +44887,7 @@ L_AES_ARM64_NEON_td DCB 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 DCB 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 DCB 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_NEON_shift_rows_invshuffle DCB 0x04, 0x09, 0x0e, 0x03, 0x08, 0x0d, 0x02, 0x07 @@ -44899,568 +44899,568 @@ L_AES_ARM64_NEON_shift_rows_invshuffle AES_ECB_decrypt_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x5, L_AES_ARM64_NEON_td add x5, x5, L_AES_ARM64_NEON_td adrp x6, L_AES_ARM64_NEON_shift_rows_invshuffle add x6, x6, L_AES_ARM64_NEON_shift_rows_invshuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x5], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x5], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x5], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x5], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x5], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x5], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x5] cmp x2, #0x40 blt L_AES_ECB_decrypt_NEON_start_2 L_AES_ECB_decrypt_NEON_loop_4 mov x8, x3 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - movi v28.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - ushr v14.16b, v6.16b, #6 - ushr v15.16b, v7.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - shl v2.16b, v6.16b, #2 - shl v3.16b, v7.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - ushr v2.16b, v6.16b, #5 - ushr v3.16b, v7.16b, #5 - pmul v0.16b, v0.16b, v28.16b - pmul v1.16b, v1.16b, v28.16b - pmul v2.16b, v2.16b, v28.16b - pmul v3.16b, v3.16b, v28.16b - shl v28.16b, v4.16b, #3 - shl v29.16b, v5.16b, #3 - shl v30.16b, v6.16b, #3 - shl v31.16b, v7.16b, #3 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - eor v28.16b, v8.16b, v0.16b - eor v29.16b, v9.16b, v1.16b - eor v30.16b, v10.16b, v2.16b - eor v31.16b, v11.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v10.16b, v14.16b, v2.16b - eor v11.16b, v15.16b, v3.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v4.16b - eor v29.16b, v29.16b, v5.16b - eor v30.16b, v30.16b, v6.16b - eor v31.16b, v31.16b, v7.16b - shl v4.4s, v28.4s, #8 - shl v5.4s, v29.4s, #8 - shl v6.4s, v30.4s, #8 - shl v7.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v4.4s, v28.4s, #24 - sri v5.4s, v29.4s, #24 - sri v6.4s, v30.4s, #24 - sri v7.4s, v31.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - shl v28.4s, v0.4s, #24 - shl v29.4s, v1.4s, #24 - shl v30.4s, v2.4s, #24 - shl v31.4s, v3.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - sri v28.4s, v0.4s, #8 - sri v29.4s, v1.4s, #8 - sri v30.4s, v2.4s, #8 - sri v31.4s, v3.4s, #8 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + movi V28.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + ushr V14.16B, V6.16B, #6 + ushr V15.16B, V7.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + shl V2.16B, V6.16B, #2 + shl V3.16B, V7.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + ushr V2.16B, V6.16B, #5 + ushr V3.16B, V7.16B, #5 + pmul V0.16B, V0.16B, V28.16B + pmul V1.16B, V1.16B, V28.16B + pmul V2.16B, V2.16B, V28.16B + pmul V3.16B, V3.16B, V28.16B + shl V28.16B, V4.16B, #3 + shl V29.16B, V5.16B, #3 + shl V30.16B, V6.16B, #3 + shl V31.16B, V7.16B, #3 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + eor V28.16B, V8.16B, V0.16B + eor V29.16B, V9.16B, V1.16B + eor V30.16B, V10.16B, V2.16B + eor V31.16B, V11.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V10.16B, V14.16B, V2.16B + eor V11.16B, V15.16B, V3.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V4.16B + eor V29.16B, V29.16B, V5.16B + eor V30.16B, V30.16B, V6.16B + eor V31.16B, V31.16B, V7.16B + shl V4.4S, V28.4S, #8 + shl V5.4S, V29.4S, #8 + shl V6.4S, V30.4S, #8 + shl V7.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V4.4S, V28.4S, #24 + sri V5.4S, V29.4S, #24 + sri V6.4S, V30.4S, #24 + sri V7.4S, V31.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B + shl V28.4S, V0.4S, #24 + shl V29.4S, V1.4S, #24 + shl V30.4S, V2.4S, #24 + shl V31.4S, V3.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + sri V28.4S, V0.4S, #8 + sri V29.4S, V1.4S, #8 + sri V30.4S, V2.4S, #8 + sri V31.4S, V3.4S, #8 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x5] ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - movi v28.16b, #27 - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v0.16b, #6 - ushr v13.16b, v1.16b, #6 - ushr v14.16b, v2.16b, #6 - ushr v15.16b, v3.16b, #6 - shl v4.16b, v0.16b, #2 - shl v5.16b, v1.16b, #2 - shl v6.16b, v2.16b, #2 - shl v7.16b, v3.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - eor v14.16b, v14.16b, v6.16b - eor v15.16b, v15.16b, v7.16b - ushr v4.16b, v0.16b, #5 - ushr v5.16b, v1.16b, #5 - ushr v6.16b, v2.16b, #5 - ushr v7.16b, v3.16b, #5 - pmul v4.16b, v4.16b, v28.16b - pmul v5.16b, v5.16b, v28.16b - pmul v6.16b, v6.16b, v28.16b - pmul v7.16b, v7.16b, v28.16b - shl v28.16b, v0.16b, #3 - shl v29.16b, v1.16b, #3 - shl v30.16b, v2.16b, #3 - shl v31.16b, v3.16b, #3 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - eor v28.16b, v8.16b, v4.16b - eor v29.16b, v9.16b, v5.16b - eor v30.16b, v10.16b, v6.16b - eor v31.16b, v11.16b, v7.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - eor v8.16b, v12.16b, v4.16b - eor v9.16b, v13.16b, v5.16b - eor v10.16b, v14.16b, v6.16b - eor v11.16b, v15.16b, v7.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v0.16b - eor v29.16b, v29.16b, v1.16b - eor v30.16b, v30.16b, v2.16b - eor v31.16b, v31.16b, v3.16b - shl v0.4s, v28.4s, #8 - shl v1.4s, v29.4s, #8 - shl v2.4s, v30.4s, #8 - shl v3.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v0.4s, v28.4s, #24 - sri v1.4s, v29.4s, #24 - sri v2.4s, v30.4s, #24 - sri v3.4s, v31.4s, #24 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - shl v28.4s, v4.4s, #24 - shl v29.4s, v5.4s, #24 - shl v30.4s, v6.4s, #24 - shl v31.4s, v7.4s, #24 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - sri v28.4s, v4.4s, #8 - sri v29.4s, v5.4s, #8 - sri v30.4s, v6.4s, #8 - sri v31.4s, v7.4s, #8 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + movi V28.16B, #27 + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V0.16B, #6 + ushr V13.16B, V1.16B, #6 + ushr V14.16B, V2.16B, #6 + ushr V15.16B, V3.16B, #6 + shl V4.16B, V0.16B, #2 + shl V5.16B, V1.16B, #2 + shl V6.16B, V2.16B, #2 + shl V7.16B, V3.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + eor V14.16B, V14.16B, V6.16B + eor V15.16B, V15.16B, V7.16B + ushr V4.16B, V0.16B, #5 + ushr V5.16B, V1.16B, #5 + ushr V6.16B, V2.16B, #5 + ushr V7.16B, V3.16B, #5 + pmul V4.16B, V4.16B, V28.16B + pmul V5.16B, V5.16B, V28.16B + pmul V6.16B, V6.16B, V28.16B + pmul V7.16B, V7.16B, V28.16B + shl V28.16B, V0.16B, #3 + shl V29.16B, V1.16B, #3 + shl V30.16B, V2.16B, #3 + shl V31.16B, V3.16B, #3 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + eor V28.16B, V8.16B, V4.16B + eor V29.16B, V9.16B, V5.16B + eor V30.16B, V10.16B, V6.16B + eor V31.16B, V11.16B, V7.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B + eor V8.16B, V12.16B, V4.16B + eor V9.16B, V13.16B, V5.16B + eor V10.16B, V14.16B, V6.16B + eor V11.16B, V15.16B, V7.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V0.16B + eor V29.16B, V29.16B, V1.16B + eor V30.16B, V30.16B, V2.16B + eor V31.16B, V31.16B, V3.16B + shl V0.4S, V28.4S, #8 + shl V1.4S, V29.4S, #8 + shl V2.4S, V30.4S, #8 + shl V3.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V0.4S, V28.4S, #24 + sri V1.4S, V29.4S, #24 + sri V2.4S, V30.4S, #24 + sri V3.4S, V31.4S, #24 + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B + shl V28.4S, V4.4S, #24 + shl V29.4S, V5.4S, #24 + shl V30.4S, V6.4S, #24 + shl V31.4S, V7.4S, #24 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + sri V28.4S, V4.4S, #8 + sri V29.4S, V5.4S, #8 + sri V30.4S, V6.4S, #8 + sri V31.4S, V7.4S, #8 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x5] ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - movi v28.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - ushr v14.16b, v6.16b, #6 - ushr v15.16b, v7.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - shl v2.16b, v6.16b, #2 - shl v3.16b, v7.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - ushr v2.16b, v6.16b, #5 - ushr v3.16b, v7.16b, #5 - pmul v0.16b, v0.16b, v28.16b - pmul v1.16b, v1.16b, v28.16b - pmul v2.16b, v2.16b, v28.16b - pmul v3.16b, v3.16b, v28.16b - shl v28.16b, v4.16b, #3 - shl v29.16b, v5.16b, #3 - shl v30.16b, v6.16b, #3 - shl v31.16b, v7.16b, #3 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - eor v28.16b, v8.16b, v0.16b - eor v29.16b, v9.16b, v1.16b - eor v30.16b, v10.16b, v2.16b - eor v31.16b, v11.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v10.16b, v14.16b, v2.16b - eor v11.16b, v15.16b, v3.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v4.16b - eor v29.16b, v29.16b, v5.16b - eor v30.16b, v30.16b, v6.16b - eor v31.16b, v31.16b, v7.16b - shl v4.4s, v28.4s, #8 - shl v5.4s, v29.4s, #8 - shl v6.4s, v30.4s, #8 - shl v7.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v4.4s, v28.4s, #24 - sri v5.4s, v29.4s, #24 - sri v6.4s, v30.4s, #24 - sri v7.4s, v31.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - shl v28.4s, v0.4s, #24 - shl v29.4s, v1.4s, #24 - shl v30.4s, v2.4s, #24 - shl v31.4s, v3.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - sri v28.4s, v0.4s, #8 - sri v29.4s, v1.4s, #8 - sri v30.4s, v2.4s, #8 - sri v31.4s, v3.4s, #8 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x5] + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + movi V28.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + ushr V14.16B, V6.16B, #6 + ushr V15.16B, V7.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + shl V2.16B, V6.16B, #2 + shl V3.16B, V7.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + ushr V2.16B, V6.16B, #5 + ushr V3.16B, V7.16B, #5 + pmul V0.16B, V0.16B, V28.16B + pmul V1.16B, V1.16B, V28.16B + pmul V2.16B, V2.16B, V28.16B + pmul V3.16B, V3.16B, V28.16B + shl V28.16B, V4.16B, #3 + shl V29.16B, V5.16B, #3 + shl V30.16B, V6.16B, #3 + shl V31.16B, V7.16B, #3 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + eor V28.16B, V8.16B, V0.16B + eor V29.16B, V9.16B, V1.16B + eor V30.16B, V10.16B, V2.16B + eor V31.16B, V11.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V10.16B, V14.16B, V2.16B + eor V11.16B, V15.16B, V3.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V4.16B + eor V29.16B, V29.16B, V5.16B + eor V30.16B, V30.16B, V6.16B + eor V31.16B, V31.16B, V7.16B + shl V4.4S, V28.4S, #8 + shl V5.4S, V29.4S, #8 + shl V6.4S, V30.4S, #8 + shl V7.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V4.4S, V28.4S, #24 + sri V5.4S, V29.4S, #24 + sri V6.4S, V30.4S, #24 + sri V7.4S, V31.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B + shl V28.4S, V0.4S, #24 + shl V29.4S, V1.4S, #24 + shl V30.4S, V2.4S, #24 + shl V31.4S, V3.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + sri V28.4S, V0.4S, #8 + sri V29.4S, V1.4S, #8 + sri V30.4S, V2.4S, #8 + sri V31.4S, V3.4S, #8 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x5] ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_ECB_decrypt_NEON_loop_4 @@ -45470,454 +45470,454 @@ L_AES_ECB_decrypt_NEON_start_2 blt L_AES_ECB_decrypt_NEON_data_done L_AES_ECB_decrypt_NEON_loop_2 mov x8, x3 - ld1 {v0.16b, v1.16b}, [x0], #32 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b + ld1 {V0.16B, V1.16B}, [x0], #32 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - movi v10.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - pmul v0.16b, v0.16b, v10.16b - pmul v1.16b, v1.16b, v10.16b - shl v10.16b, v4.16b, #3 - shl v11.16b, v5.16b, #3 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b - eor v10.16b, v8.16b, v0.16b - eor v11.16b, v9.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v5.16b - shl v4.4s, v10.4s, #8 - shl v5.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v4.4s, v10.4s, #24 - sri v5.4s, v11.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - shl v10.4s, v0.4s, #24 - shl v11.4s, v1.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - sri v10.4s, v0.4s, #8 - sri v11.4s, v1.4s, #8 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + movi V10.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + pmul V0.16B, V0.16B, V10.16B + pmul V1.16B, V1.16B, V10.16B + shl V10.16B, V4.16B, #3 + shl V11.16B, V5.16B, #3 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B + eor V10.16B, V8.16B, V0.16B + eor V11.16B, V9.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V5.16B + shl V4.4S, V10.4S, #8 + shl V5.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V4.4S, V10.4S, #24 + sri V5.4S, V11.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + shl V10.4S, V0.4S, #24 + shl V11.4S, V1.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + sri V10.4S, V0.4S, #8 + sri V11.4S, V1.4S, #8 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - movi v10.16b, #27 - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v0.16b, #6 - ushr v13.16b, v1.16b, #6 - shl v4.16b, v0.16b, #2 - shl v5.16b, v1.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - ushr v4.16b, v0.16b, #5 - ushr v5.16b, v1.16b, #5 - pmul v4.16b, v4.16b, v10.16b - pmul v5.16b, v5.16b, v10.16b - shl v10.16b, v0.16b, #3 - shl v11.16b, v1.16b, #3 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b - eor v10.16b, v8.16b, v4.16b - eor v11.16b, v9.16b, v5.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v8.16b, v12.16b, v4.16b - eor v9.16b, v13.16b, v5.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v0.16b - eor v11.16b, v11.16b, v1.16b - shl v0.4s, v10.4s, #8 - shl v1.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v0.4s, v10.4s, #24 - sri v1.4s, v11.4s, #24 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - shl v10.4s, v4.4s, #24 - shl v11.4s, v5.4s, #24 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - sri v10.4s, v4.4s, #8 - sri v11.4s, v5.4s, #8 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + movi V10.16B, #27 + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V0.16B, #6 + ushr V13.16B, V1.16B, #6 + shl V4.16B, V0.16B, #2 + shl V5.16B, V1.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + ushr V4.16B, V0.16B, #5 + ushr V5.16B, V1.16B, #5 + pmul V4.16B, V4.16B, V10.16B + pmul V5.16B, V5.16B, V10.16B + shl V10.16B, V0.16B, #3 + shl V11.16B, V1.16B, #3 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B + eor V10.16B, V8.16B, V4.16B + eor V11.16B, V9.16B, V5.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V8.16B, V12.16B, V4.16B + eor V9.16B, V13.16B, V5.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V0.16B + eor V11.16B, V11.16B, V1.16B + shl V0.4S, V10.4S, #8 + shl V1.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V0.4S, V10.4S, #24 + sri V1.4S, V11.4S, #24 + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + shl V10.4S, V4.4S, #24 + shl V11.4S, V5.4S, #24 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + sri V10.4S, V4.4S, #8 + sri V11.4S, V5.4S, #8 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x6] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - movi v10.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - pmul v0.16b, v0.16b, v10.16b - pmul v1.16b, v1.16b, v10.16b - shl v10.16b, v4.16b, #3 - shl v11.16b, v5.16b, #3 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b - eor v10.16b, v8.16b, v0.16b - eor v11.16b, v9.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v5.16b - shl v4.4s, v10.4s, #8 - shl v5.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v4.4s, v10.4s, #24 - sri v5.4s, v11.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - shl v10.4s, v0.4s, #24 - shl v11.4s, v1.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - sri v10.4s, v0.4s, #8 - sri v11.4s, v1.4s, #8 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x6] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + movi V10.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + pmul V0.16B, V0.16B, V10.16B + pmul V1.16B, V1.16B, V10.16B + shl V10.16B, V4.16B, #3 + shl V11.16B, V5.16B, #3 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B + eor V10.16B, V8.16B, V0.16B + eor V11.16B, V9.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V5.16B + shl V4.4S, V10.4S, #8 + shl V5.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V4.4S, V10.4S, #24 + sri V5.4S, V11.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + shl V10.4S, V0.4S, #24 + shl V11.4S, V1.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + sri V10.4S, V0.4S, #8 + sri V11.4S, V1.4S, #8 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x8], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b + ld1 {V0.2D}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x6] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x6] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x8], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x8], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + st1 {V0.16B, V1.16B}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_ECB_decrypt_NEON_data_done L_AES_ECB_decrypt_NEON_start_1 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 - ld1 {v3.2d}, [x6] + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 + ld1 {V3.2D}, [x6] mov x8, x3 - ld1 {v0.16b}, [x0], #16 - ld1 {v4.2d}, [x8], #16 - rev32 v0.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V4.2D}, [x8], #16 + rev32 V0.16B, V0.16B ; Round: 0 - XOR in key schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B sub w7, w4, #2 L_AES_ECB_decrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x8], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x8], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - sshr v10.16b, v0.16b, #7 - ushr v11.16b, v0.16b, #6 - ushr v8.16b, v0.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v0.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v0.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v0.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v0.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v0.16b - shl v0.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v0.4s, v9.4s, #24 - eor v0.16b, v0.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v0.16b, v0.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v0.16b, v0.16b, v9.16b - ld1 {v4.2d}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + sshr V10.16B, V0.16B, #7 + ushr V11.16B, V0.16B, #6 + ushr V8.16B, V0.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V0.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V0.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V0.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V0.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V0.16B + shl V0.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V0.4S, V9.4S, #24 + eor V0.16B, V0.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V0.16B, V0.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V0.16B, V0.16B, V9.16B + ld1 {V4.2D}, [x8], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B subs w7, w7, #2 bne L_AES_ECB_decrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x8], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x8], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x8], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x8], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + st1 {V0.16B}, [x1], #16 L_AES_ECB_decrypt_NEON_data_done - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -45929,578 +45929,578 @@ L_AES_ECB_decrypt_NEON_data_done AES_CBC_decrypt_NEON PROC stp x29, x30, [sp, #-160]! add x29, sp, #0 - stp d8, d9, [x29, #96] - stp d10, d11, [x29, #112] - stp d12, d13, [x29, #128] - stp d14, d15, [x29, #144] + stp D8, D9, [x29, #96] + stp D10, D11, [x29, #112] + stp D12, D13, [x29, #128] + stp D14, D15, [x29, #144] adrp x6, L_AES_ARM64_NEON_td add x6, x6, L_AES_ARM64_NEON_td adrp x7, L_AES_ARM64_NEON_shift_rows_invshuffle add x7, x7, L_AES_ARM64_NEON_shift_rows_invshuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x6], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x6], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x6], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] - ld1 {v3.2d}, [x5] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x6], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x6], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x6], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x6] + ld1 {V3.2D}, [x5] add x10, x29, #16 cmp x2, #0x40 blt L_AES_CBC_decrypt_NEON_start_2 L_AES_CBC_decrypt_NEON_loop_4 mov x9, x3 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - st1 {v3.2d, v4.2d, v5.2d, v6.2d}, [x10] - str q7, [x10, #64] - ld1 {v8.2d}, [x9], #16 - rev32 v4.16b, v4.16b - rev32 v5.16b, v5.16b - rev32 v6.16b, v6.16b - rev32 v7.16b, v7.16b + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + st1 {V3.2D, V4.2D, V5.2D, V6.2D}, [x10] + str Q7, [x10, #64] + ld1 {V8.2D}, [x9], #16 + rev32 V4.16B, V4.16B + rev32 V5.16B, V5.16B + rev32 V6.16B, V6.16B + rev32 V7.16B, V7.16B ; Round: 0 - XOR in key schedule - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B + eor V6.16B, V6.16B, V8.16B + eor V7.16B, V7.16B, V8.16B sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_4 - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v10.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v11.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v5.16b, v12.16b - eor v2.16b, v6.16b, v12.16b - eor v3.16b, v7.16b, v12.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b - tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - eor v0.16b, v4.16b, v13.16b - eor v1.16b, v5.16b, v13.16b - eor v2.16b, v6.16b, v13.16b - eor v3.16b, v7.16b, v13.16b - tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - eor v0.16b, v4.16b, v14.16b - eor v1.16b, v5.16b, v14.16b - eor v2.16b, v6.16b, v14.16b - eor v3.16b, v7.16b, v14.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld1 {v4.16b}, [x7] - tbl v8.16b, {v8.16b}, v4.16b - tbl v9.16b, {v9.16b}, v4.16b - tbl v10.16b, {v10.16b}, v4.16b - tbl v11.16b, {v11.16b}, v4.16b - movi v28.16b, #27 - sshr v0.16b, v8.16b, #7 - sshr v1.16b, v9.16b, #7 - sshr v2.16b, v10.16b, #7 - sshr v3.16b, v11.16b, #7 - shl v12.16b, v8.16b, #1 - shl v13.16b, v9.16b, #1 - shl v14.16b, v10.16b, #1 - shl v15.16b, v11.16b, #1 - and v0.16b, v0.16b, v28.16b - and v1.16b, v1.16b, v28.16b - and v2.16b, v2.16b, v28.16b - and v3.16b, v3.16b, v28.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - ushr v12.16b, v8.16b, #6 - ushr v13.16b, v9.16b, #6 - ushr v14.16b, v10.16b, #6 - ushr v15.16b, v11.16b, #6 - shl v4.16b, v8.16b, #2 - shl v5.16b, v9.16b, #2 - shl v6.16b, v10.16b, #2 - shl v7.16b, v11.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - eor v14.16b, v14.16b, v6.16b - eor v15.16b, v15.16b, v7.16b - ushr v4.16b, v8.16b, #5 - ushr v5.16b, v9.16b, #5 - ushr v6.16b, v10.16b, #5 - ushr v7.16b, v11.16b, #5 - pmul v4.16b, v4.16b, v28.16b - pmul v5.16b, v5.16b, v28.16b - pmul v6.16b, v6.16b, v28.16b - pmul v7.16b, v7.16b, v28.16b - shl v28.16b, v8.16b, #3 - shl v29.16b, v9.16b, #3 - shl v30.16b, v10.16b, #3 - shl v31.16b, v11.16b, #3 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - eor v28.16b, v0.16b, v4.16b - eor v29.16b, v1.16b, v5.16b - eor v30.16b, v2.16b, v6.16b - eor v31.16b, v3.16b, v7.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v0.16b, v12.16b, v4.16b - eor v1.16b, v13.16b, v5.16b - eor v2.16b, v14.16b, v6.16b - eor v3.16b, v15.16b, v7.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v8.16b - eor v29.16b, v29.16b, v9.16b - eor v30.16b, v30.16b, v10.16b - eor v31.16b, v31.16b, v11.16b - shl v8.4s, v28.4s, #8 - shl v9.4s, v29.4s, #8 - shl v10.4s, v30.4s, #8 - shl v11.4s, v31.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - sri v8.4s, v28.4s, #24 - sri v9.4s, v29.4s, #24 - sri v10.4s, v30.4s, #24 - sri v11.4s, v31.4s, #24 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - shl v28.4s, v4.4s, #24 - shl v29.4s, v5.4s, #24 - shl v30.4s, v6.4s, #24 - shl v31.4s, v7.4s, #24 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - sri v28.4s, v4.4s, #8 - sri v29.4s, v5.4s, #8 - sri v30.4s, v6.4s, #8 - sri v31.4s, v7.4s, #8 - eor v8.16b, v8.16b, v28.16b - eor v9.16b, v9.16b, v29.16b - eor v10.16b, v10.16b, v30.16b - eor v11.16b, v11.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V9.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V10.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V11.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V5.16B, V12.16B + eor V2.16B, V6.16B, V12.16B + eor V3.16B, V7.16B, V12.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + tbl V2.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V2.16B + tbl V3.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + eor V0.16B, V4.16B, V13.16B + eor V1.16B, V5.16B, V13.16B + eor V2.16B, V6.16B, V13.16B + eor V3.16B, V7.16B, V13.16B + tbl V0.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + eor V0.16B, V4.16B, V14.16B + eor V1.16B, V5.16B, V14.16B + eor V2.16B, V6.16B, V14.16B + eor V3.16B, V7.16B, V14.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + tbl V3.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld1 {V4.16B}, [x7] + tbl V8.16B, {V8.16B}, V4.16B + tbl V9.16B, {V9.16B}, V4.16B + tbl V10.16B, {V10.16B}, V4.16B + tbl V11.16B, {V11.16B}, V4.16B + movi V28.16B, #27 + sshr V0.16B, V8.16B, #7 + sshr V1.16B, V9.16B, #7 + sshr V2.16B, V10.16B, #7 + sshr V3.16B, V11.16B, #7 + shl V12.16B, V8.16B, #1 + shl V13.16B, V9.16B, #1 + shl V14.16B, V10.16B, #1 + shl V15.16B, V11.16B, #1 + and V0.16B, V0.16B, V28.16B + and V1.16B, V1.16B, V28.16B + and V2.16B, V2.16B, V28.16B + and V3.16B, V3.16B, V28.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B + ushr V12.16B, V8.16B, #6 + ushr V13.16B, V9.16B, #6 + ushr V14.16B, V10.16B, #6 + ushr V15.16B, V11.16B, #6 + shl V4.16B, V8.16B, #2 + shl V5.16B, V9.16B, #2 + shl V6.16B, V10.16B, #2 + shl V7.16B, V11.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + eor V14.16B, V14.16B, V6.16B + eor V15.16B, V15.16B, V7.16B + ushr V4.16B, V8.16B, #5 + ushr V5.16B, V9.16B, #5 + ushr V6.16B, V10.16B, #5 + ushr V7.16B, V11.16B, #5 + pmul V4.16B, V4.16B, V28.16B + pmul V5.16B, V5.16B, V28.16B + pmul V6.16B, V6.16B, V28.16B + pmul V7.16B, V7.16B, V28.16B + shl V28.16B, V8.16B, #3 + shl V29.16B, V9.16B, #3 + shl V30.16B, V10.16B, #3 + shl V31.16B, V11.16B, #3 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + eor V28.16B, V0.16B, V4.16B + eor V29.16B, V1.16B, V5.16B + eor V30.16B, V2.16B, V6.16B + eor V31.16B, V3.16B, V7.16B + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V0.16B, V12.16B, V4.16B + eor V1.16B, V13.16B, V5.16B + eor V2.16B, V14.16B, V6.16B + eor V3.16B, V15.16B, V7.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V8.16B + eor V29.16B, V29.16B, V9.16B + eor V30.16B, V30.16B, V10.16B + eor V31.16B, V31.16B, V11.16B + shl V8.4S, V28.4S, #8 + shl V9.4S, V29.4S, #8 + shl V10.4S, V30.4S, #8 + shl V11.4S, V31.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + sri V8.4S, V28.4S, #24 + sri V9.4S, V29.4S, #24 + sri V10.4S, V30.4S, #24 + sri V11.4S, V31.4S, #24 + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + shl V28.4S, V4.4S, #24 + shl V29.4S, V5.4S, #24 + shl V30.4S, V6.4S, #24 + shl V31.4S, V7.4S, #24 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + sri V28.4S, V4.4S, #8 + sri V29.4S, V5.4S, #8 + sri V30.4S, V6.4S, #8 + sri V31.4S, V7.4S, #8 + eor V8.16B, V8.16B, V28.16B + eor V9.16B, V9.16B, V29.16B + eor V10.16B, V10.16B, V30.16B + eor V11.16B, V11.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x6] ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v4.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V4.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V4.16B ; Round Done - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v10.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v11.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v9.16b, v12.16b - eor v2.16b, v10.16b, v12.16b - eor v3.16b, v11.16b, v12.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b - tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - eor v0.16b, v8.16b, v13.16b - eor v1.16b, v9.16b, v13.16b - eor v2.16b, v10.16b, v13.16b - eor v3.16b, v11.16b, v13.16b - tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - eor v0.16b, v8.16b, v14.16b - eor v1.16b, v9.16b, v14.16b - eor v2.16b, v10.16b, v14.16b - eor v3.16b, v11.16b, v14.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - ld1 {v8.16b}, [x7] - tbl v4.16b, {v4.16b}, v8.16b - tbl v5.16b, {v5.16b}, v8.16b - tbl v6.16b, {v6.16b}, v8.16b - tbl v7.16b, {v7.16b}, v8.16b - movi v28.16b, #27 - sshr v0.16b, v4.16b, #7 - sshr v1.16b, v5.16b, #7 - sshr v2.16b, v6.16b, #7 - sshr v3.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - and v0.16b, v0.16b, v28.16b - and v1.16b, v1.16b, v28.16b - and v2.16b, v2.16b, v28.16b - and v3.16b, v3.16b, v28.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - ushr v14.16b, v6.16b, #6 - ushr v15.16b, v7.16b, #6 - shl v8.16b, v4.16b, #2 - shl v9.16b, v5.16b, #2 - shl v10.16b, v6.16b, #2 - shl v11.16b, v7.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v8.16b - eor v13.16b, v13.16b, v9.16b - eor v14.16b, v14.16b, v10.16b - eor v15.16b, v15.16b, v11.16b - ushr v8.16b, v4.16b, #5 - ushr v9.16b, v5.16b, #5 - ushr v10.16b, v6.16b, #5 - ushr v11.16b, v7.16b, #5 - pmul v8.16b, v8.16b, v28.16b - pmul v9.16b, v9.16b, v28.16b - pmul v10.16b, v10.16b, v28.16b - pmul v11.16b, v11.16b, v28.16b - shl v28.16b, v4.16b, #3 - shl v29.16b, v5.16b, #3 - shl v30.16b, v6.16b, #3 - shl v31.16b, v7.16b, #3 - eor v8.16b, v8.16b, v28.16b - eor v9.16b, v9.16b, v29.16b - eor v10.16b, v10.16b, v30.16b - eor v11.16b, v11.16b, v31.16b - eor v28.16b, v0.16b, v8.16b - eor v29.16b, v1.16b, v9.16b - eor v30.16b, v2.16b, v10.16b - eor v31.16b, v3.16b, v11.16b - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v5.16b - eor v10.16b, v10.16b, v6.16b - eor v11.16b, v11.16b, v7.16b - eor v0.16b, v12.16b, v8.16b - eor v1.16b, v13.16b, v9.16b - eor v2.16b, v14.16b, v10.16b - eor v3.16b, v15.16b, v11.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v4.16b - eor v29.16b, v29.16b, v5.16b - eor v30.16b, v30.16b, v6.16b - eor v31.16b, v31.16b, v7.16b - shl v4.4s, v28.4s, #8 - shl v5.4s, v29.4s, #8 - shl v6.4s, v30.4s, #8 - shl v7.4s, v31.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - sri v4.4s, v28.4s, #24 - sri v5.4s, v29.4s, #24 - sri v6.4s, v30.4s, #24 - sri v7.4s, v31.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - shl v28.4s, v8.4s, #24 - shl v29.4s, v9.4s, #24 - shl v30.4s, v10.4s, #24 - shl v31.4s, v11.4s, #24 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - sri v28.4s, v8.4s, #8 - sri v29.4s, v9.4s, #8 - sri v30.4s, v10.4s, #8 - sri v31.4s, v11.4s, #8 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V9.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V10.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V11.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V9.16B, V12.16B + eor V2.16B, V10.16B, V12.16B + eor V3.16B, V11.16B, V12.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + tbl V2.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V2.16B + tbl V3.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + eor V0.16B, V8.16B, V13.16B + eor V1.16B, V9.16B, V13.16B + eor V2.16B, V10.16B, V13.16B + eor V3.16B, V11.16B, V13.16B + tbl V0.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + eor V0.16B, V8.16B, V14.16B + eor V1.16B, V9.16B, V14.16B + eor V2.16B, V10.16B, V14.16B + eor V3.16B, V11.16B, V14.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + tbl V3.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + ld1 {V8.16B}, [x7] + tbl V4.16B, {V4.16B}, V8.16B + tbl V5.16B, {V5.16B}, V8.16B + tbl V6.16B, {V6.16B}, V8.16B + tbl V7.16B, {V7.16B}, V8.16B + movi V28.16B, #27 + sshr V0.16B, V4.16B, #7 + sshr V1.16B, V5.16B, #7 + sshr V2.16B, V6.16B, #7 + sshr V3.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + and V0.16B, V0.16B, V28.16B + and V1.16B, V1.16B, V28.16B + and V2.16B, V2.16B, V28.16B + and V3.16B, V3.16B, V28.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + ushr V14.16B, V6.16B, #6 + ushr V15.16B, V7.16B, #6 + shl V8.16B, V4.16B, #2 + shl V9.16B, V5.16B, #2 + shl V10.16B, V6.16B, #2 + shl V11.16B, V7.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V8.16B + eor V13.16B, V13.16B, V9.16B + eor V14.16B, V14.16B, V10.16B + eor V15.16B, V15.16B, V11.16B + ushr V8.16B, V4.16B, #5 + ushr V9.16B, V5.16B, #5 + ushr V10.16B, V6.16B, #5 + ushr V11.16B, V7.16B, #5 + pmul V8.16B, V8.16B, V28.16B + pmul V9.16B, V9.16B, V28.16B + pmul V10.16B, V10.16B, V28.16B + pmul V11.16B, V11.16B, V28.16B + shl V28.16B, V4.16B, #3 + shl V29.16B, V5.16B, #3 + shl V30.16B, V6.16B, #3 + shl V31.16B, V7.16B, #3 + eor V8.16B, V8.16B, V28.16B + eor V9.16B, V9.16B, V29.16B + eor V10.16B, V10.16B, V30.16B + eor V11.16B, V11.16B, V31.16B + eor V28.16B, V0.16B, V8.16B + eor V29.16B, V1.16B, V9.16B + eor V30.16B, V2.16B, V10.16B + eor V31.16B, V3.16B, V11.16B + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V5.16B + eor V10.16B, V10.16B, V6.16B + eor V11.16B, V11.16B, V7.16B + eor V0.16B, V12.16B, V8.16B + eor V1.16B, V13.16B, V9.16B + eor V2.16B, V14.16B, V10.16B + eor V3.16B, V15.16B, V11.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V4.16B + eor V29.16B, V29.16B, V5.16B + eor V30.16B, V30.16B, V6.16B + eor V31.16B, V31.16B, V7.16B + shl V4.4S, V28.4S, #8 + shl V5.4S, V29.4S, #8 + shl V6.4S, V30.4S, #8 + shl V7.4S, V31.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + sri V4.4S, V28.4S, #24 + sri V5.4S, V29.4S, #24 + sri V6.4S, V30.4S, #24 + sri V7.4S, V31.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B + shl V28.4S, V8.4S, #24 + shl V29.4S, V9.4S, #24 + shl V30.4S, V10.4S, #24 + shl V31.4S, V11.4S, #24 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B + sri V28.4S, V8.4S, #8 + sri V29.4S, V9.4S, #8 + sri V30.4S, V10.4S, #8 + sri V31.4S, V11.4S, #8 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x6] ; XOR in Key Schedule - ld1 {v8.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b + ld1 {V8.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B + eor V6.16B, V6.16B, V8.16B + eor V7.16B, V7.16B, V8.16B ; Round Done subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_4 - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v10.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v11.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v5.16b, v12.16b - eor v2.16b, v6.16b, v12.16b - eor v3.16b, v7.16b, v12.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b - tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - eor v0.16b, v4.16b, v13.16b - eor v1.16b, v5.16b, v13.16b - eor v2.16b, v6.16b, v13.16b - eor v3.16b, v7.16b, v13.16b - tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - eor v0.16b, v4.16b, v14.16b - eor v1.16b, v5.16b, v14.16b - eor v2.16b, v6.16b, v14.16b - eor v3.16b, v7.16b, v14.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld1 {v4.16b}, [x7] - tbl v8.16b, {v8.16b}, v4.16b - tbl v9.16b, {v9.16b}, v4.16b - tbl v10.16b, {v10.16b}, v4.16b - tbl v11.16b, {v11.16b}, v4.16b - movi v28.16b, #27 - sshr v0.16b, v8.16b, #7 - sshr v1.16b, v9.16b, #7 - sshr v2.16b, v10.16b, #7 - sshr v3.16b, v11.16b, #7 - shl v12.16b, v8.16b, #1 - shl v13.16b, v9.16b, #1 - shl v14.16b, v10.16b, #1 - shl v15.16b, v11.16b, #1 - and v0.16b, v0.16b, v28.16b - and v1.16b, v1.16b, v28.16b - and v2.16b, v2.16b, v28.16b - and v3.16b, v3.16b, v28.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - ushr v12.16b, v8.16b, #6 - ushr v13.16b, v9.16b, #6 - ushr v14.16b, v10.16b, #6 - ushr v15.16b, v11.16b, #6 - shl v4.16b, v8.16b, #2 - shl v5.16b, v9.16b, #2 - shl v6.16b, v10.16b, #2 - shl v7.16b, v11.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - eor v14.16b, v14.16b, v6.16b - eor v15.16b, v15.16b, v7.16b - ushr v4.16b, v8.16b, #5 - ushr v5.16b, v9.16b, #5 - ushr v6.16b, v10.16b, #5 - ushr v7.16b, v11.16b, #5 - pmul v4.16b, v4.16b, v28.16b - pmul v5.16b, v5.16b, v28.16b - pmul v6.16b, v6.16b, v28.16b - pmul v7.16b, v7.16b, v28.16b - shl v28.16b, v8.16b, #3 - shl v29.16b, v9.16b, #3 - shl v30.16b, v10.16b, #3 - shl v31.16b, v11.16b, #3 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - eor v28.16b, v0.16b, v4.16b - eor v29.16b, v1.16b, v5.16b - eor v30.16b, v2.16b, v6.16b - eor v31.16b, v3.16b, v7.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v0.16b, v12.16b, v4.16b - eor v1.16b, v13.16b, v5.16b - eor v2.16b, v14.16b, v6.16b - eor v3.16b, v15.16b, v7.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v8.16b - eor v29.16b, v29.16b, v9.16b - eor v30.16b, v30.16b, v10.16b - eor v31.16b, v31.16b, v11.16b - shl v8.4s, v28.4s, #8 - shl v9.4s, v29.4s, #8 - shl v10.4s, v30.4s, #8 - shl v11.4s, v31.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - sri v8.4s, v28.4s, #24 - sri v9.4s, v29.4s, #24 - sri v10.4s, v30.4s, #24 - sri v11.4s, v31.4s, #24 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - shl v28.4s, v4.4s, #24 - shl v29.4s, v5.4s, #24 - shl v30.4s, v6.4s, #24 - shl v31.4s, v7.4s, #24 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - eor v10.16b, v10.16b, v2.16b - eor v11.16b, v11.16b, v3.16b - sri v28.4s, v4.4s, #8 - sri v29.4s, v5.4s, #8 - sri v30.4s, v6.4s, #8 - sri v31.4s, v7.4s, #8 - eor v8.16b, v8.16b, v28.16b - eor v9.16b, v9.16b, v29.16b - eor v10.16b, v10.16b, v30.16b - eor v11.16b, v11.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x6] + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V9.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V10.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V11.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V5.16B, V12.16B + eor V2.16B, V6.16B, V12.16B + eor V3.16B, V7.16B, V12.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + tbl V2.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V2.16B + tbl V3.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + eor V0.16B, V4.16B, V13.16B + eor V1.16B, V5.16B, V13.16B + eor V2.16B, V6.16B, V13.16B + eor V3.16B, V7.16B, V13.16B + tbl V0.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + eor V0.16B, V4.16B, V14.16B + eor V1.16B, V5.16B, V14.16B + eor V2.16B, V6.16B, V14.16B + eor V3.16B, V7.16B, V14.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + tbl V3.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld1 {V4.16B}, [x7] + tbl V8.16B, {V8.16B}, V4.16B + tbl V9.16B, {V9.16B}, V4.16B + tbl V10.16B, {V10.16B}, V4.16B + tbl V11.16B, {V11.16B}, V4.16B + movi V28.16B, #27 + sshr V0.16B, V8.16B, #7 + sshr V1.16B, V9.16B, #7 + sshr V2.16B, V10.16B, #7 + sshr V3.16B, V11.16B, #7 + shl V12.16B, V8.16B, #1 + shl V13.16B, V9.16B, #1 + shl V14.16B, V10.16B, #1 + shl V15.16B, V11.16B, #1 + and V0.16B, V0.16B, V28.16B + and V1.16B, V1.16B, V28.16B + and V2.16B, V2.16B, V28.16B + and V3.16B, V3.16B, V28.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B + ushr V12.16B, V8.16B, #6 + ushr V13.16B, V9.16B, #6 + ushr V14.16B, V10.16B, #6 + ushr V15.16B, V11.16B, #6 + shl V4.16B, V8.16B, #2 + shl V5.16B, V9.16B, #2 + shl V6.16B, V10.16B, #2 + shl V7.16B, V11.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + eor V14.16B, V14.16B, V6.16B + eor V15.16B, V15.16B, V7.16B + ushr V4.16B, V8.16B, #5 + ushr V5.16B, V9.16B, #5 + ushr V6.16B, V10.16B, #5 + ushr V7.16B, V11.16B, #5 + pmul V4.16B, V4.16B, V28.16B + pmul V5.16B, V5.16B, V28.16B + pmul V6.16B, V6.16B, V28.16B + pmul V7.16B, V7.16B, V28.16B + shl V28.16B, V8.16B, #3 + shl V29.16B, V9.16B, #3 + shl V30.16B, V10.16B, #3 + shl V31.16B, V11.16B, #3 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + eor V28.16B, V0.16B, V4.16B + eor V29.16B, V1.16B, V5.16B + eor V30.16B, V2.16B, V6.16B + eor V31.16B, V3.16B, V7.16B + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V0.16B, V12.16B, V4.16B + eor V1.16B, V13.16B, V5.16B + eor V2.16B, V14.16B, V6.16B + eor V3.16B, V15.16B, V7.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V8.16B + eor V29.16B, V29.16B, V9.16B + eor V30.16B, V30.16B, V10.16B + eor V31.16B, V31.16B, V11.16B + shl V8.4S, V28.4S, #8 + shl V9.4S, V29.4S, #8 + shl V10.4S, V30.4S, #8 + shl V11.4S, V31.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + sri V8.4S, V28.4S, #24 + sri V9.4S, V29.4S, #24 + sri V10.4S, V30.4S, #24 + sri V11.4S, V31.4S, #24 + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + shl V28.4S, V4.4S, #24 + shl V29.4S, V5.4S, #24 + shl V30.4S, V6.4S, #24 + shl V31.4S, V7.4S, #24 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + eor V10.16B, V10.16B, V2.16B + eor V11.16B, V11.16B, V3.16B + sri V28.4S, V4.4S, #8 + sri V29.4S, V5.4S, #8 + sri V30.4S, V6.4S, #8 + sri V31.4S, V7.4S, #8 + eor V8.16B, V8.16B, V28.16B + eor V9.16B, V9.16B, V29.16B + eor V10.16B, V10.16B, V30.16B + eor V11.16B, V11.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x6] ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v4.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V4.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V4.16B ; Round Done - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v10.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v11.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v9.16b, v12.16b - eor v2.16b, v10.16b, v12.16b - eor v3.16b, v11.16b, v12.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - tbl v2.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v2.16b - tbl v3.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - eor v0.16b, v8.16b, v13.16b - eor v1.16b, v9.16b, v13.16b - eor v2.16b, v10.16b, v13.16b - eor v3.16b, v11.16b, v13.16b - tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - eor v0.16b, v8.16b, v14.16b - eor v1.16b, v9.16b, v14.16b - eor v2.16b, v10.16b, v14.16b - eor v3.16b, v11.16b, v14.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - tbl v3.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - orr v6.16b, v6.16b, v2.16b - orr v7.16b, v7.16b, v3.16b - ld1 {v8.16b}, [x7] - tbl v4.16b, {v4.16b}, v8.16b - tbl v5.16b, {v5.16b}, v8.16b - tbl v6.16b, {v6.16b}, v8.16b - tbl v7.16b, {v7.16b}, v8.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V9.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V10.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V11.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V9.16B, V12.16B + eor V2.16B, V10.16B, V12.16B + eor V3.16B, V11.16B, V12.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + tbl V2.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V2.16B + tbl V3.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + eor V0.16B, V8.16B, V13.16B + eor V1.16B, V9.16B, V13.16B + eor V2.16B, V10.16B, V13.16B + eor V3.16B, V11.16B, V13.16B + tbl V0.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + eor V0.16B, V8.16B, V14.16B + eor V1.16B, V9.16B, V14.16B + eor V2.16B, V10.16B, V14.16B + eor V3.16B, V11.16B, V14.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + tbl V3.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + orr V6.16B, V6.16B, V2.16B + orr V7.16B, V7.16B, V3.16B + ld1 {V8.16B}, [x7] + tbl V4.16B, {V4.16B}, V8.16B + tbl V5.16B, {V5.16B}, V8.16B + tbl V6.16B, {V6.16B}, V8.16B + tbl V7.16B, {V7.16B}, V8.16B ; XOR in Key Schedule - ld1 {v8.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b - eor v6.16b, v6.16b, v8.16b - eor v7.16b, v7.16b, v8.16b + ld1 {V8.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B + eor V6.16B, V6.16B, V8.16B + eor V7.16B, V7.16B, V8.16B ; Round Done - rev32 v4.16b, v4.16b - rev32 v5.16b, v5.16b - rev32 v6.16b, v6.16b - rev32 v7.16b, v7.16b - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x10] - ldr q3, [x10, #64] - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + rev32 V4.16B, V4.16B + rev32 V5.16B, V5.16B + rev32 V6.16B, V6.16B + rev32 V7.16B, V7.16B + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x10] + ldr Q3, [x10, #64] + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + st1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_CBC_decrypt_NEON_loop_4 @@ -46510,465 +46510,465 @@ L_AES_CBC_decrypt_NEON_start_2 blt L_AES_CBC_decrypt_NEON_data_done L_AES_CBC_decrypt_NEON_loop_2 mov x9, x3 - ld1 {v4.16b, v5.16b}, [x0], #32 - st1 {v3.2d, v4.2d, v5.2d}, [x10] - ld1 {v8.2d}, [x9], #16 - rev32 v4.16b, v4.16b - rev32 v5.16b, v5.16b + ld1 {V4.16B, V5.16B}, [x0], #32 + st1 {V3.2D, V4.2D, V5.2D}, [x10] + ld1 {V8.2D}, [x9], #16 + rev32 V4.16B, V4.16B + rev32 V5.16B, V5.16B ; Round: 0 - XOR in key schedule - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v5.16b, v12.16b - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - eor v2.16b, v4.16b, v13.16b - eor v3.16b, v5.16b, v13.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - eor v0.16b, v4.16b, v14.16b - eor v1.16b, v5.16b, v14.16b - orr v8.16b, v8.16b, v2.16b - orr v9.16b, v9.16b, v3.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - ld1 {v4.16b}, [x7] - tbl v8.16b, {v8.16b}, v4.16b - tbl v9.16b, {v9.16b}, v4.16b - movi v2.16b, #27 - sshr v0.16b, v8.16b, #7 - sshr v1.16b, v9.16b, #7 - shl v12.16b, v8.16b, #1 - shl v13.16b, v9.16b, #1 - and v0.16b, v0.16b, v2.16b - and v1.16b, v1.16b, v2.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - ushr v12.16b, v8.16b, #6 - ushr v13.16b, v9.16b, #6 - shl v4.16b, v8.16b, #2 - shl v5.16b, v9.16b, #2 - pmul v12.16b, v12.16b, v2.16b - pmul v13.16b, v13.16b, v2.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - ushr v4.16b, v8.16b, #5 - ushr v5.16b, v9.16b, #5 - pmul v4.16b, v4.16b, v2.16b - pmul v5.16b, v5.16b, v2.16b - shl v2.16b, v8.16b, #3 - shl v3.16b, v9.16b, #3 - eor v4.16b, v4.16b, v2.16b - eor v5.16b, v5.16b, v3.16b - eor v2.16b, v0.16b, v4.16b - eor v3.16b, v1.16b, v5.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v0.16b, v12.16b, v4.16b - eor v1.16b, v13.16b, v5.16b - eor v12.16b, v12.16b, v2.16b - eor v13.16b, v13.16b, v3.16b - eor v2.16b, v2.16b, v8.16b - eor v3.16b, v3.16b, v9.16b - shl v8.4s, v2.4s, #8 - shl v9.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - sri v8.4s, v2.4s, #24 - sri v9.4s, v3.4s, #24 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - shl v2.4s, v4.4s, #24 - shl v3.4s, v5.4s, #24 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - sri v2.4s, v4.4s, #8 - sri v3.4s, v5.4s, #8 - eor v8.16b, v8.16b, v2.16b - eor v9.16b, v9.16b, v3.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V5.16B, V12.16B + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V9.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + eor V2.16B, V4.16B, V13.16B + eor V3.16B, V5.16B, V13.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + eor V0.16B, V4.16B, V14.16B + eor V1.16B, V5.16B, V14.16B + orr V8.16B, V8.16B, V2.16B + orr V9.16B, V9.16B, V3.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + ld1 {V4.16B}, [x7] + tbl V8.16B, {V8.16B}, V4.16B + tbl V9.16B, {V9.16B}, V4.16B + movi V2.16B, #27 + sshr V0.16B, V8.16B, #7 + sshr V1.16B, V9.16B, #7 + shl V12.16B, V8.16B, #1 + shl V13.16B, V9.16B, #1 + and V0.16B, V0.16B, V2.16B + and V1.16B, V1.16B, V2.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + ushr V12.16B, V8.16B, #6 + ushr V13.16B, V9.16B, #6 + shl V4.16B, V8.16B, #2 + shl V5.16B, V9.16B, #2 + pmul V12.16B, V12.16B, V2.16B + pmul V13.16B, V13.16B, V2.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + ushr V4.16B, V8.16B, #5 + ushr V5.16B, V9.16B, #5 + pmul V4.16B, V4.16B, V2.16B + pmul V5.16B, V5.16B, V2.16B + shl V2.16B, V8.16B, #3 + shl V3.16B, V9.16B, #3 + eor V4.16B, V4.16B, V2.16B + eor V5.16B, V5.16B, V3.16B + eor V2.16B, V0.16B, V4.16B + eor V3.16B, V1.16B, V5.16B + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V0.16B, V12.16B, V4.16B + eor V1.16B, V13.16B, V5.16B + eor V12.16B, V12.16B, V2.16B + eor V13.16B, V13.16B, V3.16B + eor V2.16B, V2.16B, V8.16B + eor V3.16B, V3.16B, V9.16B + shl V8.4S, V2.4S, #8 + shl V9.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + sri V8.4S, V2.4S, #24 + sri V9.4S, V3.4S, #24 + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + shl V2.4S, V4.4S, #24 + shl V3.4S, V5.4S, #24 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + sri V2.4S, V4.4S, #8 + sri V3.4S, V5.4S, #8 + eor V8.16B, V8.16B, V2.16B + eor V9.16B, V9.16B, V3.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V4.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v9.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - eor v2.16b, v8.16b, v13.16b - eor v3.16b, v9.16b, v13.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - eor v0.16b, v8.16b, v14.16b - eor v1.16b, v9.16b, v14.16b - orr v4.16b, v4.16b, v2.16b - orr v5.16b, v5.16b, v3.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - ld1 {v8.16b}, [x7] - tbl v4.16b, {v4.16b}, v8.16b - tbl v5.16b, {v5.16b}, v8.16b - movi v2.16b, #27 - sshr v0.16b, v4.16b, #7 - sshr v1.16b, v5.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - and v0.16b, v0.16b, v2.16b - and v1.16b, v1.16b, v2.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - shl v8.16b, v4.16b, #2 - shl v9.16b, v5.16b, #2 - pmul v12.16b, v12.16b, v2.16b - pmul v13.16b, v13.16b, v2.16b - eor v12.16b, v12.16b, v8.16b - eor v13.16b, v13.16b, v9.16b - ushr v8.16b, v4.16b, #5 - ushr v9.16b, v5.16b, #5 - pmul v8.16b, v8.16b, v2.16b - pmul v9.16b, v9.16b, v2.16b - shl v2.16b, v4.16b, #3 - shl v3.16b, v5.16b, #3 - eor v8.16b, v8.16b, v2.16b - eor v9.16b, v9.16b, v3.16b - eor v2.16b, v0.16b, v8.16b - eor v3.16b, v1.16b, v9.16b - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v5.16b - eor v0.16b, v12.16b, v8.16b - eor v1.16b, v13.16b, v9.16b - eor v12.16b, v12.16b, v2.16b - eor v13.16b, v13.16b, v3.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v5.16b - shl v4.4s, v2.4s, #8 - shl v5.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - sri v4.4s, v2.4s, #24 - sri v5.4s, v3.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - shl v2.4s, v8.4s, #24 - shl v3.4s, v9.4s, #24 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - sri v2.4s, v8.4s, #8 - sri v3.4s, v9.4s, #8 - eor v4.16b, v4.16b, v2.16b - eor v5.16b, v5.16b, v3.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V9.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V9.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + eor V2.16B, V8.16B, V13.16B + eor V3.16B, V9.16B, V13.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + eor V0.16B, V8.16B, V14.16B + eor V1.16B, V9.16B, V14.16B + orr V4.16B, V4.16B, V2.16B + orr V5.16B, V5.16B, V3.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + ld1 {V8.16B}, [x7] + tbl V4.16B, {V4.16B}, V8.16B + tbl V5.16B, {V5.16B}, V8.16B + movi V2.16B, #27 + sshr V0.16B, V4.16B, #7 + sshr V1.16B, V5.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + and V0.16B, V0.16B, V2.16B + and V1.16B, V1.16B, V2.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + shl V8.16B, V4.16B, #2 + shl V9.16B, V5.16B, #2 + pmul V12.16B, V12.16B, V2.16B + pmul V13.16B, V13.16B, V2.16B + eor V12.16B, V12.16B, V8.16B + eor V13.16B, V13.16B, V9.16B + ushr V8.16B, V4.16B, #5 + ushr V9.16B, V5.16B, #5 + pmul V8.16B, V8.16B, V2.16B + pmul V9.16B, V9.16B, V2.16B + shl V2.16B, V4.16B, #3 + shl V3.16B, V5.16B, #3 + eor V8.16B, V8.16B, V2.16B + eor V9.16B, V9.16B, V3.16B + eor V2.16B, V0.16B, V8.16B + eor V3.16B, V1.16B, V9.16B + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V5.16B + eor V0.16B, V12.16B, V8.16B + eor V1.16B, V13.16B, V9.16B + eor V12.16B, V12.16B, V2.16B + eor V13.16B, V13.16B, V3.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V5.16B + shl V4.4S, V2.4S, #8 + shl V5.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + sri V4.4S, V2.4S, #24 + sri V5.4S, V3.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + shl V2.4S, V8.4S, #24 + shl V3.4S, V9.4S, #24 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + sri V2.4S, V8.4S, #8 + sri V3.4S, V9.4S, #8 + eor V4.16B, V4.16B, V2.16B + eor V5.16B, V5.16B, V3.16B ; XOR in Key Schedule - ld1 {v8.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b + ld1 {V8.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B ; Round Done subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v5.16b, v12.16b - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v9.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - eor v2.16b, v4.16b, v13.16b - eor v3.16b, v5.16b, v13.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - eor v0.16b, v4.16b, v14.16b - eor v1.16b, v5.16b, v14.16b - orr v8.16b, v8.16b, v2.16b - orr v9.16b, v9.16b, v3.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - ld1 {v4.16b}, [x7] - tbl v8.16b, {v8.16b}, v4.16b - tbl v9.16b, {v9.16b}, v4.16b - movi v2.16b, #27 - sshr v0.16b, v8.16b, #7 - sshr v1.16b, v9.16b, #7 - shl v12.16b, v8.16b, #1 - shl v13.16b, v9.16b, #1 - and v0.16b, v0.16b, v2.16b - and v1.16b, v1.16b, v2.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - ushr v12.16b, v8.16b, #6 - ushr v13.16b, v9.16b, #6 - shl v4.16b, v8.16b, #2 - shl v5.16b, v9.16b, #2 - pmul v12.16b, v12.16b, v2.16b - pmul v13.16b, v13.16b, v2.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - ushr v4.16b, v8.16b, #5 - ushr v5.16b, v9.16b, #5 - pmul v4.16b, v4.16b, v2.16b - pmul v5.16b, v5.16b, v2.16b - shl v2.16b, v8.16b, #3 - shl v3.16b, v9.16b, #3 - eor v4.16b, v4.16b, v2.16b - eor v5.16b, v5.16b, v3.16b - eor v2.16b, v0.16b, v4.16b - eor v3.16b, v1.16b, v5.16b - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v0.16b, v12.16b, v4.16b - eor v1.16b, v13.16b, v5.16b - eor v12.16b, v12.16b, v2.16b - eor v13.16b, v13.16b, v3.16b - eor v2.16b, v2.16b, v8.16b - eor v3.16b, v3.16b, v9.16b - shl v8.4s, v2.4s, #8 - shl v9.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - sri v8.4s, v2.4s, #24 - sri v9.4s, v3.4s, #24 - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - shl v2.4s, v4.4s, #24 - shl v3.4s, v5.4s, #24 - eor v8.16b, v8.16b, v0.16b - eor v9.16b, v9.16b, v1.16b - sri v2.4s, v4.4s, #8 - sri v3.4s, v5.4s, #8 - eor v8.16b, v8.16b, v2.16b - eor v9.16b, v9.16b, v3.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V5.16B, V12.16B + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V9.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + eor V2.16B, V4.16B, V13.16B + eor V3.16B, V5.16B, V13.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + eor V0.16B, V4.16B, V14.16B + eor V1.16B, V5.16B, V14.16B + orr V8.16B, V8.16B, V2.16B + orr V9.16B, V9.16B, V3.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + ld1 {V4.16B}, [x7] + tbl V8.16B, {V8.16B}, V4.16B + tbl V9.16B, {V9.16B}, V4.16B + movi V2.16B, #27 + sshr V0.16B, V8.16B, #7 + sshr V1.16B, V9.16B, #7 + shl V12.16B, V8.16B, #1 + shl V13.16B, V9.16B, #1 + and V0.16B, V0.16B, V2.16B + and V1.16B, V1.16B, V2.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + ushr V12.16B, V8.16B, #6 + ushr V13.16B, V9.16B, #6 + shl V4.16B, V8.16B, #2 + shl V5.16B, V9.16B, #2 + pmul V12.16B, V12.16B, V2.16B + pmul V13.16B, V13.16B, V2.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + ushr V4.16B, V8.16B, #5 + ushr V5.16B, V9.16B, #5 + pmul V4.16B, V4.16B, V2.16B + pmul V5.16B, V5.16B, V2.16B + shl V2.16B, V8.16B, #3 + shl V3.16B, V9.16B, #3 + eor V4.16B, V4.16B, V2.16B + eor V5.16B, V5.16B, V3.16B + eor V2.16B, V0.16B, V4.16B + eor V3.16B, V1.16B, V5.16B + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V0.16B, V12.16B, V4.16B + eor V1.16B, V13.16B, V5.16B + eor V12.16B, V12.16B, V2.16B + eor V13.16B, V13.16B, V3.16B + eor V2.16B, V2.16B, V8.16B + eor V3.16B, V3.16B, V9.16B + shl V8.4S, V2.4S, #8 + shl V9.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + sri V8.4S, V2.4S, #24 + sri V9.4S, V3.4S, #24 + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + shl V2.4S, V4.4S, #24 + shl V3.4S, V5.4S, #24 + eor V8.16B, V8.16B, V0.16B + eor V9.16B, V9.16B, V1.16B + sri V2.4S, V4.4S, #8 + sri V3.4S, V5.4S, #8 + eor V8.16B, V8.16B, V2.16B + eor V9.16B, V9.16B, V3.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x9], #16 - eor v8.16b, v8.16b, v4.16b - eor v9.16b, v9.16b, v4.16b + ld1 {V4.2D}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V9.16B, V9.16B, V4.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v9.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v9.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v1.16b - eor v2.16b, v8.16b, v13.16b - eor v3.16b, v9.16b, v13.16b - tbl v2.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v2.16b - tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - eor v0.16b, v8.16b, v14.16b - eor v1.16b, v9.16b, v14.16b - orr v4.16b, v4.16b, v2.16b - orr v5.16b, v5.16b, v3.16b - tbl v0.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v0.16b - tbl v1.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v1.16b - orr v4.16b, v4.16b, v0.16b - orr v5.16b, v5.16b, v1.16b - ld1 {v8.16b}, [x7] - tbl v4.16b, {v4.16b}, v8.16b - tbl v5.16b, {v5.16b}, v8.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V9.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V9.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V1.16B + eor V2.16B, V8.16B, V13.16B + eor V3.16B, V9.16B, V13.16B + tbl V2.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V2.16B + tbl V3.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V3.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + eor V0.16B, V8.16B, V14.16B + eor V1.16B, V9.16B, V14.16B + orr V4.16B, V4.16B, V2.16B + orr V5.16B, V5.16B, V3.16B + tbl V0.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V0.16B + tbl V1.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V1.16B + orr V4.16B, V4.16B, V0.16B + orr V5.16B, V5.16B, V1.16B + ld1 {V8.16B}, [x7] + tbl V4.16B, {V4.16B}, V8.16B + tbl V5.16B, {V5.16B}, V8.16B ; XOR in Key Schedule - ld1 {v8.2d}, [x9], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v8.16b + ld1 {V8.2D}, [x9], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V8.16B ; Round Done - rev32 v4.16b, v4.16b - rev32 v5.16b, v5.16b - ld1 {v1.16b, v2.16b, v3.16b}, [x10] - eor v4.16b, v4.16b, v1.16b - eor v5.16b, v5.16b, v2.16b - st1 {v4.16b, v5.16b}, [x1], #32 + rev32 V4.16B, V4.16B + rev32 V5.16B, V5.16B + ld1 {V1.16B, V2.16B, V3.16B}, [x10] + eor V4.16B, V4.16B, V1.16B + eor V5.16B, V5.16B, V2.16B + st1 {V4.16B, V5.16B}, [x1], #32 sub x2, x2, #32 cmp x2, #32 bge L_AES_CBC_decrypt_NEON_loop_2 cmp x2, #0 beq L_AES_CBC_decrypt_NEON_data_done L_AES_CBC_decrypt_NEON_start_1 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 - ld1 {v7.2d}, [x7] + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 + ld1 {V7.2D}, [x7] mov x9, x3 - ld1 {v4.16b}, [x0], #16 - mov v10.16b, v3.16b - mov v11.16b, v4.16b - ld1 {v8.16b}, [x9], #16 - rev32 v4.16b, v4.16b + ld1 {V4.16B}, [x0], #16 + mov V10.16B, V3.16B + mov V11.16B, V4.16B + ld1 {V8.16B}, [x9], #16 + rev32 V4.16B, V4.16B ; Round: 0 - XOR in key schedule - eor v4.16b, v4.16b, v8.16b + eor V4.16B, V4.16B, V8.16B sub w8, w4, #2 L_AES_CBC_decrypt_NEON_loop_nr_1 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v4.16b, v13.16b - eor v2.16b, v4.16b, v14.16b - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - orr v8.16b, v8.16b, v0.16b - orr v1.16b, v1.16b, v2.16b - orr v8.16b, v8.16b, v1.16b - tbl v8.16b, {v8.16b}, v7.16b - sshr v2.16b, v8.16b, #7 - ushr v3.16b, v8.16b, #6 - ushr v0.16b, v8.16b, #5 - and v2.16b, v2.16b, v15.16b - pmul v3.16b, v3.16b, v15.16b - pmul v0.16b, v0.16b, v15.16b - shl v1.16b, v8.16b, #1 - eor v2.16b, v2.16b, v1.16b - shl v1.16b, v8.16b, #3 - eor v0.16b, v0.16b, v1.16b - shl v1.16b, v8.16b, #2 - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v2.16b, v0.16b - eor v0.16b, v0.16b, v8.16b - eor v2.16b, v3.16b, v0.16b - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v1.16b, v8.16b - shl v8.4s, v1.4s, #8 - rev32 v2.8h, v2.8h - sri v8.4s, v1.4s, #24 - eor v8.16b, v8.16b, v3.16b - shl v1.4s, v0.4s, #24 - eor v8.16b, v8.16b, v2.16b - sri v1.4s, v0.4s, #8 - eor v8.16b, v8.16b, v1.16b - ld1 {v4.2d}, [x9], #16 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V4.16B, V13.16B + eor V2.16B, V4.16B, V14.16B + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + orr V8.16B, V8.16B, V0.16B + orr V1.16B, V1.16B, V2.16B + orr V8.16B, V8.16B, V1.16B + tbl V8.16B, {V8.16B}, V7.16B + sshr V2.16B, V8.16B, #7 + ushr V3.16B, V8.16B, #6 + ushr V0.16B, V8.16B, #5 + and V2.16B, V2.16B, V15.16B + pmul V3.16B, V3.16B, V15.16B + pmul V0.16B, V0.16B, V15.16B + shl V1.16B, V8.16B, #1 + eor V2.16B, V2.16B, V1.16B + shl V1.16B, V8.16B, #3 + eor V0.16B, V0.16B, V1.16B + shl V1.16B, V8.16B, #2 + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V2.16B, V0.16B + eor V0.16B, V0.16B, V8.16B + eor V2.16B, V3.16B, V0.16B + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V1.16B, V8.16B + shl V8.4S, V1.4S, #8 + rev32 V2.8H, V2.8H + sri V8.4S, V1.4S, #24 + eor V8.16B, V8.16B, V3.16B + shl V1.4S, V0.4S, #24 + eor V8.16B, V8.16B, V2.16B + sri V1.4S, V0.4S, #8 + eor V8.16B, V8.16B, V1.16B + ld1 {V4.2D}, [x9], #16 ; XOR in Key Schedule - eor v8.16b, v8.16b, v4.16b - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v8.16b, v13.16b - eor v2.16b, v8.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - orr v4.16b, v4.16b, v0.16b - orr v1.16b, v1.16b, v2.16b - orr v4.16b, v4.16b, v1.16b - tbl v4.16b, {v4.16b}, v7.16b - sshr v2.16b, v4.16b, #7 - ushr v3.16b, v4.16b, #6 - ushr v0.16b, v4.16b, #5 - and v2.16b, v2.16b, v15.16b - pmul v3.16b, v3.16b, v15.16b - pmul v0.16b, v0.16b, v15.16b - shl v1.16b, v4.16b, #1 - eor v2.16b, v2.16b, v1.16b - shl v1.16b, v4.16b, #3 - eor v0.16b, v0.16b, v1.16b - shl v1.16b, v4.16b, #2 - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v2.16b, v0.16b - eor v0.16b, v0.16b, v4.16b - eor v2.16b, v3.16b, v0.16b - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v1.16b, v4.16b - shl v4.4s, v1.4s, #8 - rev32 v2.8h, v2.8h - sri v4.4s, v1.4s, #24 - eor v4.16b, v4.16b, v3.16b - shl v1.4s, v0.4s, #24 - eor v4.16b, v4.16b, v2.16b - sri v1.4s, v0.4s, #8 - eor v4.16b, v4.16b, v1.16b - ld1 {v8.2d}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V8.16B, V13.16B + eor V2.16B, V8.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + orr V4.16B, V4.16B, V0.16B + orr V1.16B, V1.16B, V2.16B + orr V4.16B, V4.16B, V1.16B + tbl V4.16B, {V4.16B}, V7.16B + sshr V2.16B, V4.16B, #7 + ushr V3.16B, V4.16B, #6 + ushr V0.16B, V4.16B, #5 + and V2.16B, V2.16B, V15.16B + pmul V3.16B, V3.16B, V15.16B + pmul V0.16B, V0.16B, V15.16B + shl V1.16B, V4.16B, #1 + eor V2.16B, V2.16B, V1.16B + shl V1.16B, V4.16B, #3 + eor V0.16B, V0.16B, V1.16B + shl V1.16B, V4.16B, #2 + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V2.16B, V0.16B + eor V0.16B, V0.16B, V4.16B + eor V2.16B, V3.16B, V0.16B + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V1.16B, V4.16B + shl V4.4S, V1.4S, #8 + rev32 V2.8H, V2.8H + sri V4.4S, V1.4S, #24 + eor V4.16B, V4.16B, V3.16B + shl V1.4S, V0.4S, #24 + eor V4.16B, V4.16B, V2.16B + sri V1.4S, V0.4S, #8 + eor V4.16B, V4.16B, V1.16B + ld1 {V8.2D}, [x9], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v8.16b + eor V4.16B, V4.16B, V8.16B subs w8, w8, #2 bne L_AES_CBC_decrypt_NEON_loop_nr_1 - eor v0.16b, v4.16b, v12.16b - eor v1.16b, v4.16b, v13.16b - eor v2.16b, v4.16b, v14.16b - tbl v8.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - orr v8.16b, v8.16b, v0.16b - orr v1.16b, v1.16b, v2.16b - orr v8.16b, v8.16b, v1.16b - tbl v8.16b, {v8.16b}, v7.16b - sshr v2.16b, v8.16b, #7 - ushr v3.16b, v8.16b, #6 - ushr v0.16b, v8.16b, #5 - and v2.16b, v2.16b, v15.16b - pmul v3.16b, v3.16b, v15.16b - pmul v0.16b, v0.16b, v15.16b - shl v1.16b, v8.16b, #1 - eor v2.16b, v2.16b, v1.16b - shl v1.16b, v8.16b, #3 - eor v0.16b, v0.16b, v1.16b - shl v1.16b, v8.16b, #2 - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v2.16b, v0.16b - eor v0.16b, v0.16b, v8.16b - eor v2.16b, v3.16b, v0.16b - eor v3.16b, v3.16b, v1.16b - eor v1.16b, v1.16b, v8.16b - shl v8.4s, v1.4s, #8 - rev32 v2.8h, v2.8h - sri v8.4s, v1.4s, #24 - eor v8.16b, v8.16b, v3.16b - shl v1.4s, v0.4s, #24 - eor v8.16b, v8.16b, v2.16b - sri v1.4s, v0.4s, #8 - eor v8.16b, v8.16b, v1.16b - ld1 {v4.2d}, [x9], #16 + eor V0.16B, V4.16B, V12.16B + eor V1.16B, V4.16B, V13.16B + eor V2.16B, V4.16B, V14.16B + tbl V8.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + orr V8.16B, V8.16B, V0.16B + orr V1.16B, V1.16B, V2.16B + orr V8.16B, V8.16B, V1.16B + tbl V8.16B, {V8.16B}, V7.16B + sshr V2.16B, V8.16B, #7 + ushr V3.16B, V8.16B, #6 + ushr V0.16B, V8.16B, #5 + and V2.16B, V2.16B, V15.16B + pmul V3.16B, V3.16B, V15.16B + pmul V0.16B, V0.16B, V15.16B + shl V1.16B, V8.16B, #1 + eor V2.16B, V2.16B, V1.16B + shl V1.16B, V8.16B, #3 + eor V0.16B, V0.16B, V1.16B + shl V1.16B, V8.16B, #2 + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V2.16B, V0.16B + eor V0.16B, V0.16B, V8.16B + eor V2.16B, V3.16B, V0.16B + eor V3.16B, V3.16B, V1.16B + eor V1.16B, V1.16B, V8.16B + shl V8.4S, V1.4S, #8 + rev32 V2.8H, V2.8H + sri V8.4S, V1.4S, #24 + eor V8.16B, V8.16B, V3.16B + shl V1.4S, V0.4S, #24 + eor V8.16B, V8.16B, V2.16B + sri V1.4S, V0.4S, #8 + eor V8.16B, V8.16B, V1.16B + ld1 {V4.2D}, [x9], #16 ; XOR in Key Schedule - eor v8.16b, v8.16b, v4.16b - eor v0.16b, v8.16b, v12.16b - eor v1.16b, v8.16b, v13.16b - eor v2.16b, v8.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v8.16b - tbl v0.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v0.16b - tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b - tbl v2.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v2.16b - orr v4.16b, v4.16b, v0.16b - orr v1.16b, v1.16b, v2.16b - orr v4.16b, v4.16b, v1.16b - tbl v4.16b, {v4.16b}, v7.16b - ld1 {v8.2d}, [x9], #16 + eor V8.16B, V8.16B, V4.16B + eor V0.16B, V8.16B, V12.16B + eor V1.16B, V8.16B, V13.16B + eor V2.16B, V8.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V8.16B + tbl V0.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V0.16B + tbl V1.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V1.16B + tbl V2.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V2.16B + orr V4.16B, V4.16B, V0.16B + orr V1.16B, V1.16B, V2.16B + orr V4.16B, V4.16B, V1.16B + tbl V4.16B, {V4.16B}, V7.16B + ld1 {V8.2D}, [x9], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v8.16b - rev32 v4.16b, v4.16b - mov v3.16b, v11.16b - eor v4.16b, v4.16b, v10.16b - st1 {v4.16b}, [x1], #16 + eor V4.16B, V4.16B, V8.16B + rev32 V4.16B, V4.16B + mov V3.16B, V11.16B + eor V4.16B, V4.16B, V10.16B + st1 {V4.16B}, [x1], #16 L_AES_CBC_decrypt_NEON_data_done - st1 {v3.2d}, [x5] - ldp d8, d9, [x29, #96] - ldp d10, d11, [x29, #112] - ldp d12, d13, [x29, #128] - ldp d14, d15, [x29, #144] + st1 {V3.2D}, [x5] + ldp D8, D9, [x29, #96] + ldp D10, D11, [x29, #112] + ldp D12, D13, [x29, #128] + ldp D14, D15, [x29, #144] ldp x29, x30, [sp], #0xa0 ret ENDP @@ -46982,329 +46982,329 @@ L_AES_CBC_decrypt_NEON_data_done GCM_gmult_len_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] - ld1 {v18.2d}, [x0] - ld1 {v10.2d}, [x1] - movi v19.16b, #15 - eor v20.16b, v20.16b, v20.16b - rbit v18.16b, v18.16b - rbit v10.16b, v10.16b - and v12.16b, v10.16b, v19.16b - ushr v13.16b, v10.16b, #4 - eor v14.16b, v12.16b, v13.16b + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + ld1 {V18.2D}, [x0] + ld1 {V10.2D}, [x1] + movi V19.16B, #15 + eor V20.16B, V20.16B, V20.16B + rbit V18.16B, V18.16B + rbit V10.16B, V10.16B + and V12.16B, V10.16B, V19.16B + ushr V13.16B, V10.16B, #4 + eor V14.16B, V12.16B, V13.16B L_GCM_gmult_len_NEON_start_block - ld1 {v0.16b}, [x2], #16 - rbit v0.16b, v0.16b - eor v18.16b, v18.16b, v0.16b + ld1 {V0.16B}, [x2], #16 + rbit V0.16B, V0.16B + eor V18.16B, V18.16B, V0.16B ; Mul 128x128 - and v15.16b, v18.16b, v19.16b - ushr v16.16b, v18.16b, #4 - eor v17.16b, v15.16b, v16.16b - dup v0.16b, v12.b[0] - dup v2.16b, v14.b[0] - dup v1.16b, v13.b[0] - pmul v8.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v8.16b - eor v5.16b, v5.16b, v4.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v8.16b, v8.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - dup v0.16b, v12.b[1] - dup v2.16b, v14.b[1] - dup v1.16b, v13.b[1] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v6.16b, v20.16b, v3.16b, #15 - ext v9.16b, v3.16b, v20.16b, #15 - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[2] - dup v2.16b, v14.b[2] - dup v1.16b, v13.b[2] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #14 - ext v6.16b, v20.16b, v3.16b, #14 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[3] - dup v2.16b, v14.b[3] - dup v1.16b, v13.b[3] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #13 - ext v6.16b, v20.16b, v3.16b, #13 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[4] - dup v2.16b, v14.b[4] - dup v1.16b, v13.b[4] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #12 - ext v6.16b, v20.16b, v3.16b, #12 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[5] - dup v2.16b, v14.b[5] - dup v1.16b, v13.b[5] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #11 - ext v6.16b, v20.16b, v3.16b, #11 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[6] - dup v2.16b, v14.b[6] - dup v1.16b, v13.b[6] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #10 - ext v6.16b, v20.16b, v3.16b, #10 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[7] - dup v2.16b, v14.b[7] - dup v1.16b, v13.b[7] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #9 - ext v6.16b, v20.16b, v3.16b, #9 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[8] - dup v2.16b, v14.b[8] - dup v1.16b, v13.b[8] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #8 - ext v6.16b, v20.16b, v3.16b, #8 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[9] - dup v2.16b, v14.b[9] - dup v1.16b, v13.b[9] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #7 - ext v6.16b, v20.16b, v3.16b, #7 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[10] - dup v2.16b, v14.b[10] - dup v1.16b, v13.b[10] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #6 - ext v6.16b, v20.16b, v3.16b, #6 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[11] - dup v2.16b, v14.b[11] - dup v1.16b, v13.b[11] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #5 - ext v6.16b, v20.16b, v3.16b, #5 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[12] - dup v2.16b, v14.b[12] - dup v1.16b, v13.b[12] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #4 - ext v6.16b, v20.16b, v3.16b, #4 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[13] - dup v2.16b, v14.b[13] - dup v1.16b, v13.b[13] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #3 - ext v6.16b, v20.16b, v3.16b, #3 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[14] - dup v2.16b, v14.b[14] - dup v1.16b, v13.b[14] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #2 - ext v6.16b, v20.16b, v3.16b, #2 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - dup v0.16b, v12.b[15] - dup v2.16b, v14.b[15] - dup v1.16b, v13.b[15] - pmul v3.16b, v15.16b, v0.16b - pmul v5.16b, v17.16b, v2.16b - pmul v4.16b, v16.16b, v1.16b - eor v5.16b, v5.16b, v3.16b - eor v5.16b, v5.16b, v4.16b - eor v3.16b, v3.16b, v11.16b - shl v6.16b, v5.16b, #4 - ushr v7.16b, v5.16b, #4 - eor v3.16b, v3.16b, v6.16b - eor v11.16b, v4.16b, v7.16b - ext v7.16b, v3.16b, v20.16b, #1 - ext v6.16b, v20.16b, v3.16b, #1 - eor v9.16b, v9.16b, v7.16b - eor v8.16b, v8.16b, v6.16b - eor v9.16b, v9.16b, v11.16b + and V15.16B, V18.16B, V19.16B + ushr V16.16B, V18.16B, #4 + eor V17.16B, V15.16B, V16.16B + dup V0.16B, V12.B[0] + dup V2.16B, V14.B[0] + dup V1.16B, V13.B[0] + pmul V8.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V8.16B + eor V5.16B, V5.16B, V4.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V8.16B, V8.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + dup V0.16B, V12.B[1] + dup V2.16B, V14.B[1] + dup V1.16B, V13.B[1] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V6.16B, V20.16B, V3.16B, #15 + ext8 V9.16B, V3.16B, V20.16B, #15 + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[2] + dup V2.16B, V14.B[2] + dup V1.16B, V13.B[2] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #14 + ext8 V6.16B, V20.16B, V3.16B, #14 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[3] + dup V2.16B, V14.B[3] + dup V1.16B, V13.B[3] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #13 + ext8 V6.16B, V20.16B, V3.16B, #13 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[4] + dup V2.16B, V14.B[4] + dup V1.16B, V13.B[4] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #12 + ext8 V6.16B, V20.16B, V3.16B, #12 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[5] + dup V2.16B, V14.B[5] + dup V1.16B, V13.B[5] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #11 + ext8 V6.16B, V20.16B, V3.16B, #11 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[6] + dup V2.16B, V14.B[6] + dup V1.16B, V13.B[6] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #10 + ext8 V6.16B, V20.16B, V3.16B, #10 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[7] + dup V2.16B, V14.B[7] + dup V1.16B, V13.B[7] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #9 + ext8 V6.16B, V20.16B, V3.16B, #9 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[8] + dup V2.16B, V14.B[8] + dup V1.16B, V13.B[8] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #8 + ext8 V6.16B, V20.16B, V3.16B, #8 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[9] + dup V2.16B, V14.B[9] + dup V1.16B, V13.B[9] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #7 + ext8 V6.16B, V20.16B, V3.16B, #7 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[10] + dup V2.16B, V14.B[10] + dup V1.16B, V13.B[10] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #6 + ext8 V6.16B, V20.16B, V3.16B, #6 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[11] + dup V2.16B, V14.B[11] + dup V1.16B, V13.B[11] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #5 + ext8 V6.16B, V20.16B, V3.16B, #5 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[12] + dup V2.16B, V14.B[12] + dup V1.16B, V13.B[12] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #4 + ext8 V6.16B, V20.16B, V3.16B, #4 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[13] + dup V2.16B, V14.B[13] + dup V1.16B, V13.B[13] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #3 + ext8 V6.16B, V20.16B, V3.16B, #3 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[14] + dup V2.16B, V14.B[14] + dup V1.16B, V13.B[14] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #2 + ext8 V6.16B, V20.16B, V3.16B, #2 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + dup V0.16B, V12.B[15] + dup V2.16B, V14.B[15] + dup V1.16B, V13.B[15] + pmul V3.16B, V15.16B, V0.16B + pmul V5.16B, V17.16B, V2.16B + pmul V4.16B, V16.16B, V1.16B + eor V5.16B, V5.16B, V3.16B + eor V5.16B, V5.16B, V4.16B + eor V3.16B, V3.16B, V11.16B + shl V6.16B, V5.16B, #4 + ushr V7.16B, V5.16B, #4 + eor V3.16B, V3.16B, V6.16B + eor V11.16B, V4.16B, V7.16B + ext8 V7.16B, V3.16B, V20.16B, #1 + ext8 V6.16B, V20.16B, V3.16B, #1 + eor V9.16B, V9.16B, V7.16B + eor V8.16B, V8.16B, V6.16B + eor V9.16B, V9.16B, V11.16B ; Reduce 254-bit number - shl v0.16b, v9.16b, #1 - shl v1.16b, v9.16b, #2 - shl v2.16b, v9.16b, #7 - ushr v3.16b, v9.16b, #7 - ushr v4.16b, v9.16b, #6 - ushr v5.16b, v9.16b, #1 - eor v0.16b, v0.16b, v9.16b - eor v1.16b, v1.16b, v2.16b - eor v0.16b, v0.16b, v1.16b - eor v8.16b, v8.16b, v0.16b - ext v0.16b, v20.16b, v3.16b, #15 - ext v1.16b, v20.16b, v4.16b, #15 - ext v2.16b, v20.16b, v5.16b, #15 - ext v4.16b, v4.16b, v20.16b, #15 - ext v5.16b, v5.16b, v20.16b, #15 - eor v0.16b, v0.16b, v1.16b - eor v8.16b, v8.16b, v2.16b - eor v8.16b, v8.16b, v0.16b - eor v3.16b, v4.16b, v5.16b - shl v0.2d, v3.2d, #1 - shl v1.2d, v3.2d, #2 - shl v2.2d, v3.2d, #7 - eor v3.16b, v3.16b, v0.16b - eor v1.16b, v1.16b, v2.16b - eor v8.16b, v8.16b, v3.16b - eor v18.16b, v8.16b, v1.16b + shl V0.16B, V9.16B, #1 + shl V1.16B, V9.16B, #2 + shl V2.16B, V9.16B, #7 + ushr V3.16B, V9.16B, #7 + ushr V4.16B, V9.16B, #6 + ushr V5.16B, V9.16B, #1 + eor V0.16B, V0.16B, V9.16B + eor V1.16B, V1.16B, V2.16B + eor V0.16B, V0.16B, V1.16B + eor V8.16B, V8.16B, V0.16B + ext8 V0.16B, V20.16B, V3.16B, #15 + ext8 V1.16B, V20.16B, V4.16B, #15 + ext8 V2.16B, V20.16B, V5.16B, #15 + ext8 V4.16B, V4.16B, V20.16B, #15 + ext8 V5.16B, V5.16B, V20.16B, #15 + eor V0.16B, V0.16B, V1.16B + eor V8.16B, V8.16B, V2.16B + eor V8.16B, V8.16B, V0.16B + eor V3.16B, V4.16B, V5.16B + shl V0.2D, V3.2D, #1 + shl V1.2D, V3.2D, #2 + shl V2.2D, V3.2D, #7 + eor V3.16B, V3.16B, V0.16B + eor V1.16B, V1.16B, V2.16B + eor V8.16B, V8.16B, V3.16B + eor V18.16B, V8.16B, V1.16B subs x3, x3, #16 bne L_GCM_gmult_len_NEON_start_block - rbit v18.16b, v18.16b - st1 {v18.2d}, [x0] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + rbit V18.16B, V18.16B + st1 {V18.2D}, [x0] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -47314,781 +47314,781 @@ L_GCM_gmult_len_NEON_start_block AES_GCM_encrypt_NEON PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x9, L_AES_ARM64_NEON_te add x9, x9, L_AES_ARM64_NEON_te adrp x10, L_AES_ARM64_NEON_shift_rows_shuffle add x10, x10, L_AES_ARM64_NEON_shift_rows_shuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x9], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x9], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x9], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x9] - ld1 {v2.2d}, [x5] - rev32 v2.16b, v2.16b - mov w6, v2.s[3] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x9], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x9], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x9], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x9] + ld1 {V2.2D}, [x5] + rev32 V2.16B, V2.16B + mov w6, V2.S[3] cmp x2, #0x40 blt L_AES_GCM_encrypt_NEON_start_2 - mov x7, v2.d[0] - mov x8, v2.d[1] + mov x7, V2.D[0] + mov x8, V2.D[1] L_AES_GCM_encrypt_NEON_loop_4 mov x12, x3 - ld1 {v4.2d}, [x12], #16 - mov v8.d[0], x7 - mov v8.d[1], x8 + ld1 {V4.2D}, [x12], #16 + mov V8.D[0], x7 + mov V8.D[1], x8 ; Round: 0 - XOR in key schedule add w6, w6, #1 - mov v8.s[3], w6 - eor v0.16b, v8.16b, v4.16b + mov V8.S[3], w6 + eor V0.16B, V8.16B, V4.16B add w6, w6, #1 - mov v8.s[3], w6 - eor v1.16b, v8.16b, v4.16b + mov V8.S[3], w6 + eor V1.16B, V8.16B, V4.16B add w6, w6, #1 - mov v8.s[3], w6 - eor v2.16b, v8.16b, v4.16b + mov V8.S[3], w6 + eor V2.16B, V8.16B, V4.16B add w6, w6, #1 - mov v8.s[3], w6 - eor v3.16b, v8.16b, v4.16b + mov V8.S[3], w6 + eor V3.16B, V8.16B, V4.16B sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x10] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x10] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x12], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x12], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x10] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - movi v4.16b, #27 - and v8.16b, v8.16b, v4.16b - and v9.16b, v9.16b, v4.16b - and v10.16b, v10.16b, v4.16b - and v11.16b, v11.16b, v4.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - eor v6.16b, v10.16b, v2.16b - eor v7.16b, v11.16b, v3.16b - shl v12.4s, v4.4s, #8 - shl v13.4s, v5.4s, #8 - shl v14.4s, v6.4s, #8 - shl v15.4s, v7.4s, #8 - sri v12.4s, v4.4s, #24 - sri v13.4s, v5.4s, #24 - sri v14.4s, v6.4s, #24 - sri v15.4s, v7.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - shl v6.4s, v2.4s, #24 - shl v7.4s, v3.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - sri v6.4s, v2.4s, #8 - sri v7.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x10] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + movi V4.16B, #27 + and V8.16B, V8.16B, V4.16B + and V9.16B, V9.16B, V4.16B + and V10.16B, V10.16B, V4.16B + and V11.16B, V11.16B, V4.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + eor V6.16B, V10.16B, V2.16B + eor V7.16B, V11.16B, V3.16B + shl V12.4S, V4.4S, #8 + shl V13.4S, V5.4S, #8 + shl V14.4S, V6.4S, #8 + shl V15.4S, V7.4S, #8 + sri V12.4S, V4.4S, #24 + sri V13.4S, V5.4S, #24 + sri V14.4S, V6.4S, #24 + sri V15.4S, V7.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + shl V6.4S, V2.4S, #24 + shl V7.4S, V3.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + sri V6.4S, V2.4S, #8 + sri V7.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x12], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b + ld1 {V4.2D}, [x12], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B ; Round Done subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x10] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x10] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x12], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x12], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x10] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x10] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x12], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x12], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x0], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 sub x2, x2, #0x40 cmp x2, #0x40 bge L_AES_GCM_encrypt_NEON_loop_4 - mov v2.d[0], x7 - mov v2.d[1], x8 - mov v2.s[3], w6 + mov V2.D[0], x7 + mov V2.D[1], x8 + mov V2.S[3], w6 L_AES_GCM_encrypt_NEON_start_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 cmp x2, #16 beq L_AES_GCM_encrypt_NEON_start_1 blt L_AES_GCM_encrypt_NEON_data_done L_AES_GCM_encrypt_NEON_loop_2 mov x12, x3 - ld1 {v4.2d}, [x12], #16 + ld1 {V4.2D}, [x12], #16 ; Round: 0 - XOR in key schedule add w6, w6, #1 - mov v2.s[3], w6 - eor v0.16b, v2.16b, v4.16b + mov V2.S[3], w6 + eor V0.16B, V2.16B, V4.16B add w6, w6, #1 - mov v2.s[3], w6 - eor v1.16b, v2.16b, v4.16b + mov V2.S[3], w6 + eor V1.16B, V2.16B, V4.16B sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x10] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x10] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x12], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x12], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x10] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v10.16b, v0.16b, #1 - shl v11.16b, v1.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - shl v10.4s, v4.4s, #8 - shl v11.4s, v5.4s, #8 - sri v10.4s, v4.4s, #24 - sri v11.4s, v5.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x10] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V10.16B, V0.16B, #1 + shl V11.16B, V1.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + shl V10.4S, V4.4S, #8 + shl V11.4S, V5.4S, #8 + sri V10.4S, V4.4S, #24 + sri V11.4S, V5.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x12], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + ld1 {V4.2D}, [x12], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; Round Done subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x10] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x10] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x12], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x12], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x10] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x10] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x12], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x12], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - ld1 {v4.16b, v5.16b}, [x0], #32 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + ld1 {V4.16B, V5.16B}, [x0], #32 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + st1 {V0.16B, V1.16B}, [x1], #32 sub x2, x2, #32 cmp x2, #0 beq L_AES_GCM_encrypt_NEON_data_done L_AES_GCM_encrypt_NEON_start_1 - ld1 {v3.2d}, [x10] + ld1 {V3.2D}, [x10] mov x12, x3 add w6, w6, #1 - ld1 {v4.2d}, [x12], #16 - mov v2.s[3], w6 + ld1 {V4.2D}, [x12], #16 + mov V2.S[3], w6 ; Round: 0 - XOR in key schedule - eor v0.16b, v2.16b, v4.16b + eor V0.16B, V2.16B, V4.16B sub w11, w4, #2 L_AES_GCM_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x12], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x12], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x12], #16 - sshr v10.16b, v0.16b, #7 - shl v9.16b, v0.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v0.8h - eor v11.16b, v10.16b, v0.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v0.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x12], #16 + sshr V10.16B, V0.16B, #7 + shl V9.16B, V0.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V0.8H + eor V11.16B, V10.16B, V0.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V0.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v0.4s, #8 - sri v8.4s, v11.4s, #24 - eor v0.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V0.4S, #8 + sri V8.4S, V11.4S, #24 + eor V0.16B, V10.16B, V9.16B + eor V0.16B, V0.16B, V8.16B subs w11, w11, #2 bne L_AES_GCM_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x12], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x12], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x12], #16 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x12], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - ld1 {v4.16b}, [x0], #16 - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + ld1 {V4.16B}, [x0], #16 + eor V0.16B, V0.16B, V4.16B + st1 {V0.16B}, [x1], #16 L_AES_GCM_encrypt_NEON_data_done - rev32 v2.16b, v2.16b - st1 {v2.2d}, [x5] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + rev32 V2.16B, V2.16B + st1 {V2.2D}, [x5] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -48103,138 +48103,138 @@ AES_XTS_encrypt_NEON PROC stp x17, x19, [x29, #24] stp x20, x21, [x29, #40] str x22, [x29, #56] - stp d8, d9, [x29, #64] - stp d10, d11, [x29, #80] - stp d12, d13, [x29, #96] - stp d14, d15, [x29, #112] + stp D8, D9, [x29, #64] + stp D10, D11, [x29, #80] + stp D12, D13, [x29, #96] + stp D14, D15, [x29, #112] adrp x19, L_AES_ARM64_NEON_te add x19, x19, L_AES_ARM64_NEON_te adrp x20, L_AES_ARM64_NEON_shift_rows_shuffle add x20, x20, L_AES_ARM64_NEON_shift_rows_shuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x19], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x19], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x19], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x19] - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 - ld1 {v3.2d}, [x20] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x19], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x19], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x19], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x19] + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 + ld1 {V3.2D}, [x20] mov x17, #0x87 - ld1 {v2.2d}, [x3] - ld1 {v4.2d}, [x5] - rev32 v2.16b, v2.16b + ld1 {V2.2D}, [x3] + ld1 {V4.2D}, [x5] + rev32 V2.16B, V2.16B add x22, x5, #16 ; Round: 0 - XOR in key schedule - eor v2.16b, v2.16b, v4.16b + eor V2.16B, V2.16B, V4.16B sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_tweak - eor v8.16b, v2.16b, v12.16b - eor v9.16b, v2.16b, v13.16b - eor v10.16b, v2.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v2.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V2.16B, V12.16B + eor V9.16B, V2.16B, V13.16B + eor V10.16B, V2.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V2.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v2.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v2.16b, v2.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v2.16b, v2.16b, v9.16b - tbl v2.16b, {v2.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 - sshr v10.16b, v2.16b, #7 - shl v9.16b, v2.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v2.8h - eor v11.16b, v10.16b, v2.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v2.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V2.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V2.16B, V2.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V2.16B, V2.16B, V9.16B + tbl V2.16B, {V2.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 + sshr V10.16B, V2.16B, #7 + shl V9.16B, V2.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V2.8H + eor V11.16B, V10.16B, V2.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V2.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v2.4s, #8 - sri v8.4s, v11.4s, #24 - eor v2.16b, v10.16b, v9.16b - eor v2.16b, v2.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V2.4S, #8 + sri V8.4S, V11.4S, #24 + eor V2.16B, V10.16B, V9.16B + eor V2.16B, V2.16B, V8.16B subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_tweak - eor v8.16b, v2.16b, v12.16b - eor v9.16b, v2.16b, v13.16b - eor v10.16b, v2.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v2.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V2.16B, V12.16B + eor V9.16B, V2.16B, V13.16B + eor V10.16B, V2.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V2.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v2.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v2.16b, v2.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v2.16b, v2.16b, v9.16b - tbl v2.16b, {v2.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 + eor V10.16B, V10.16B, V2.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V2.16B, V2.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V2.16B, V2.16B, V9.16B + tbl V2.16B, {V2.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 ; XOR in Key Schedule - eor v2.16b, v2.16b, v4.16b - rev32 v2.16b, v2.16b - mov x8, v2.d[0] - mov x9, v2.d[1] + eor V2.16B, V2.16B, V4.16B + rev32 V2.16B, V2.16B + mov x8, V2.D[0] + mov x9, V2.D[1] cmp w2, #0x40 blt L_AES_XTS_encrypt_NEON_start_2 L_AES_XTS_encrypt_NEON_loop_4 mov x22, x4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b}, [x22], #16 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B}, [x22], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 @@ -48244,775 +48244,775 @@ L_AES_XTS_encrypt_NEON_loop_4 and x16, x17, x13, asr 63 extr x15, x13, x12, #63 eor x14, x16, x12, lsl 1 - mov v8.d[0], x8 - mov v8.d[1], x9 - mov v9.d[0], x10 - mov v9.d[1], x11 - mov v10.d[0], x12 - mov v10.d[1], x13 - mov v11.d[0], x14 - mov v11.d[1], x15 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + mov V8.D[0], x8 + mov V8.D[1], x9 + mov V9.D[0], x10 + mov V9.D[1], x11 + mov V10.D[0], x12 + mov V10.D[1], x13 + mov V11.D[0], x14 + mov V11.D[1], x15 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x20] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x20] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x22], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x22], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x20] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - movi v4.16b, #27 - and v8.16b, v8.16b, v4.16b - and v9.16b, v9.16b, v4.16b - and v10.16b, v10.16b, v4.16b - and v11.16b, v11.16b, v4.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - eor v6.16b, v10.16b, v2.16b - eor v7.16b, v11.16b, v3.16b - shl v12.4s, v4.4s, #8 - shl v13.4s, v5.4s, #8 - shl v14.4s, v6.4s, #8 - shl v15.4s, v7.4s, #8 - sri v12.4s, v4.4s, #24 - sri v13.4s, v5.4s, #24 - sri v14.4s, v6.4s, #24 - sri v15.4s, v7.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - shl v6.4s, v2.4s, #24 - shl v7.4s, v3.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - sri v6.4s, v2.4s, #8 - sri v7.4s, v3.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - rev32 v2.8h, v2.8h - rev32 v3.8h, v3.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x20] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + movi V4.16B, #27 + and V8.16B, V8.16B, V4.16B + and V9.16B, V9.16B, V4.16B + and V10.16B, V10.16B, V4.16B + and V11.16B, V11.16B, V4.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + eor V6.16B, V10.16B, V2.16B + eor V7.16B, V11.16B, V3.16B + shl V12.4S, V4.4S, #8 + shl V13.4S, V5.4S, #8 + shl V14.4S, V6.4S, #8 + shl V15.4S, V7.4S, #8 + sri V12.4S, V4.4S, #24 + sri V13.4S, V5.4S, #24 + sri V14.4S, V6.4S, #24 + sri V15.4S, V7.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + shl V6.4S, V2.4S, #24 + shl V7.4S, V3.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + sri V6.4S, V2.4S, #8 + sri V7.4S, V3.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + rev32 V2.8H, V2.8H + rev32 V3.8H, V3.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B ; Round Done subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x20] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - movi v0.16b, #27 - and v8.16b, v8.16b, v0.16b - and v9.16b, v9.16b, v0.16b - and v10.16b, v10.16b, v0.16b - and v11.16b, v11.16b, v0.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - eor v2.16b, v10.16b, v6.16b - eor v3.16b, v11.16b, v7.16b - shl v12.4s, v0.4s, #8 - shl v13.4s, v1.4s, #8 - shl v14.4s, v2.4s, #8 - shl v15.4s, v3.4s, #8 - sri v12.4s, v0.4s, #24 - sri v13.4s, v1.4s, #24 - sri v14.4s, v2.4s, #24 - sri v15.4s, v3.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - shl v2.4s, v6.4s, #24 - shl v3.4s, v7.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - sri v2.4s, v6.4s, #8 - sri v3.4s, v7.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - rev32 v6.8h, v6.8h - rev32 v7.8h, v7.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x20] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + movi V0.16B, #27 + and V8.16B, V8.16B, V0.16B + and V9.16B, V9.16B, V0.16B + and V10.16B, V10.16B, V0.16B + and V11.16B, V11.16B, V0.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + eor V2.16B, V10.16B, V6.16B + eor V3.16B, V11.16B, V7.16B + shl V12.4S, V0.4S, #8 + shl V13.4S, V1.4S, #8 + shl V14.4S, V2.4S, #8 + shl V15.4S, V3.4S, #8 + sri V12.4S, V0.4S, #24 + sri V13.4S, V1.4S, #24 + sri V14.4S, V2.4S, #24 + sri V15.4S, V3.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + shl V2.4S, V6.4S, #24 + shl V3.4S, V7.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + sri V2.4S, V6.4S, #8 + sri V3.4S, V7.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + rev32 V6.8H, V6.8H + rev32 V7.8H, V7.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x22], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b + ld1 {V0.2D}, [x22], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x20] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x20] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - mov v8.d[0], x8 - mov v8.d[1], x9 - mov v9.d[0], x10 - mov v9.d[1], x11 - mov v10.d[0], x12 - mov v10.d[1], x13 - mov v11.d[0], x14 - mov v11.d[1], x15 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + mov V8.D[0], x8 + mov V8.D[1], x9 + mov V9.D[0], x10 + mov V9.D[1], x11 + mov V10.D[0], x12 + mov V10.D[1], x13 + mov V11.D[0], x14 + mov V11.D[1], x15 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 and x16, x17, x15, asr 63 extr x9, x15, x14, #63 eor x8, x16, x14, lsl 1 sub w2, w2, #0x40 cmp w2, #0x40 bge L_AES_XTS_encrypt_NEON_loop_4 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 L_AES_XTS_encrypt_NEON_start_2 cmp w2, #32 blt L_AES_XTS_encrypt_NEON_start_1 mov x22, x4 - ld1 {v0.16b, v1.16b}, [x0], #32 - ld1 {v4.16b}, [x22], #16 + ld1 {V0.16B, V1.16B}, [x0], #32 + ld1 {V4.16B}, [x22], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 - mov v2.d[0], x8 - mov v2.d[1], x9 - mov v3.d[0], x10 - mov v3.d[1], x11 - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v3.16b - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + mov V2.D[0], x8 + mov V2.D[1], x9 + mov V3.D[0], x10 + mov V3.D[1], x11 + eor V0.16B, V0.16B, V2.16B + eor V1.16B, V1.16B, V3.16B + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x20] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x20] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x22], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x22], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x20] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v10.16b, v0.16b, #1 - shl v11.16b, v1.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v4.16b, v8.16b, v0.16b - eor v5.16b, v9.16b, v1.16b - shl v10.4s, v4.4s, #8 - shl v11.4s, v5.4s, #8 - sri v10.4s, v4.4s, #24 - sri v11.4s, v5.4s, #24 - shl v4.4s, v0.4s, #24 - shl v5.4s, v1.4s, #24 - sri v4.4s, v0.4s, #8 - sri v5.4s, v1.4s, #8 - rev32 v0.8h, v0.8h - rev32 v1.8h, v1.8h - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x20] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V10.16B, V0.16B, #1 + shl V11.16B, V1.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V4.16B, V8.16B, V0.16B + eor V5.16B, V9.16B, V1.16B + shl V10.4S, V4.4S, #8 + shl V11.4S, V5.4S, #8 + sri V10.4S, V4.4S, #24 + sri V11.4S, V5.4S, #24 + shl V4.4S, V0.4S, #24 + shl V5.4S, V1.4S, #24 + sri V4.4S, V0.4S, #8 + sri V5.4S, V1.4S, #8 + rev32 V0.8H, V0.8H + rev32 V1.8H, V1.8H + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; Round Done subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x20] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v10.16b, v4.16b, #1 - shl v11.16b, v5.16b, #1 - and v8.16b, v8.16b, v15.16b - and v9.16b, v9.16b, v15.16b - eor v8.16b, v8.16b, v10.16b - eor v9.16b, v9.16b, v11.16b - eor v0.16b, v8.16b, v4.16b - eor v1.16b, v9.16b, v5.16b - shl v10.4s, v0.4s, #8 - shl v11.4s, v1.4s, #8 - sri v10.4s, v0.4s, #24 - sri v11.4s, v1.4s, #24 - shl v0.4s, v4.4s, #24 - shl v1.4s, v5.4s, #24 - sri v0.4s, v4.4s, #8 - sri v1.4s, v5.4s, #8 - rev32 v4.8h, v4.8h - rev32 v5.8h, v5.8h - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x20] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V10.16B, V4.16B, #1 + shl V11.16B, V5.16B, #1 + and V8.16B, V8.16B, V15.16B + and V9.16B, V9.16B, V15.16B + eor V8.16B, V8.16B, V10.16B + eor V9.16B, V9.16B, V11.16B + eor V0.16B, V8.16B, V4.16B + eor V1.16B, V9.16B, V5.16B + shl V10.4S, V0.4S, #8 + shl V11.4S, V1.4S, #8 + sri V10.4S, V0.4S, #24 + sri V11.4S, V1.4S, #24 + shl V0.4S, V4.4S, #24 + shl V1.4S, V5.4S, #24 + sri V0.4S, V4.4S, #8 + sri V1.4S, V5.4S, #8 + rev32 V4.8H, V4.8H + rev32 V5.8H, V5.8H + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x22], #16 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + ld1 {V0.2D}, [x22], #16 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; Round Done - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x20] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x20] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v3.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + eor V0.16B, V0.16B, V2.16B + eor V1.16B, V1.16B, V3.16B + st1 {V0.16B, V1.16B}, [x1], #32 and x16, x17, x11, asr 63 extr x9, x11, x10, #63 eor x8, x16, x10, lsl 1 sub w2, w2, #32 L_AES_XTS_encrypt_NEON_start_1 - ld1 {v3.2d}, [x20] - mov v2.d[0], x8 - mov v2.d[1], x9 + ld1 {V3.2D}, [x20] + mov V2.D[0], x8 + mov V2.D[1], x9 cmp w2, #16 blt L_AES_XTS_encrypt_NEON_start_partial mov x22, x4 - ld1 {v0.16b}, [x0], #16 - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v2.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v4.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V2.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V4.16B sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 - sshr v10.16b, v0.16b, #7 - shl v9.16b, v0.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v0.8h - eor v11.16b, v10.16b, v0.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v0.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 + sshr V10.16B, V0.16B, #7 + shl V9.16B, V0.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V0.8H + eor V11.16B, V10.16B, V0.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V0.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v0.4s, #8 - sri v8.4s, v11.4s, #24 - eor v0.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V0.4S, #8 + sri V8.4S, V11.4S, #24 + eor V0.16B, V10.16B, V9.16B + eor V0.16B, V0.16B, V8.16B subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v2.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V2.16B + st1 {V0.16B}, [x1], #16 subs w2, w2, #16 beq L_AES_XTS_encrypt_NEON_data_done and x16, x17, x9, asr 63 @@ -49020,12 +49020,12 @@ L_AES_XTS_encrypt_NEON_loop_nr_1 eor x8, x16, x8, lsl 1 L_AES_XTS_encrypt_NEON_start_partial cbz w2, L_AES_XTS_encrypt_NEON_data_done - mov v2.d[0], x8 - mov v2.d[1], x9 + mov V2.D[0], x8 + mov V2.D[1], x9 mov x22, x4 sub x1, x1, #16 - ld1 {v0.16b}, [x1], #16 - st1 {v0.2d}, [x6] + ld1 {V0.16B}, [x1], #16 + st1 {V0.2D}, [x6] mov w16, w2 L_AES_XTS_encrypt_NEON_start_byte ldrb w10, [x6] @@ -49037,121 +49037,121 @@ L_AES_XTS_encrypt_NEON_start_byte sub x1, x1, x2 sub x6, x6, x2 sub x1, x1, #16 - ld1 {v0.2d}, [x6] - ld1 {v4.2d}, [x22], #16 - eor v0.16b, v0.16b, v2.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v4.16b + ld1 {V0.2D}, [x6] + ld1 {V4.2D}, [x22], #16 + eor V0.16B, V0.16B, V2.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V4.16B sub w21, w7, #2 L_AES_XTS_encrypt_NEON_loop_nr_partial - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 - sshr v10.16b, v0.16b, #7 - shl v9.16b, v0.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v0.8h - eor v11.16b, v10.16b, v0.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v0.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 + sshr V10.16B, V0.16B, #7 + shl V9.16B, V0.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V0.8H + eor V11.16B, V10.16B, V0.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V0.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v0.4s, #8 - sri v8.4s, v11.4s, #24 - eor v0.16b, v10.16b, v9.16b - eor v0.16b, v0.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V0.4S, #8 + sri V8.4S, V11.4S, #24 + eor V0.16B, V10.16B, V9.16B + eor V0.16B, V0.16B, V8.16B subs w21, w21, #2 bne L_AES_XTS_encrypt_NEON_loop_nr_partial - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v0.2d}, [x22], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V0.2D}, [x22], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v0.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x22], #16 + eor V10.16B, V10.16B, V0.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x22], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v2.16b - st1 {v0.16b}, [x1] + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V2.16B + st1 {V0.16B}, [x1] L_AES_XTS_encrypt_NEON_data_done ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldr x22, [x29, #56] - ldp d8, d9, [x29, #64] - ldp d10, d11, [x29, #80] - ldp d12, d13, [x29, #96] - ldp d14, d15, [x29, #112] + ldp D8, D9, [x29, #64] + ldp D10, D11, [x29, #80] + ldp D12, D13, [x29, #96] + ldp D14, D15, [x29, #112] ldp x29, x30, [sp], #0x80 ret ENDP @@ -49166,10 +49166,10 @@ AES_XTS_decrypt_NEON PROC stp x20, x21, [x29, #32] stp x22, x23, [x29, #48] stp x24, x25, [x29, #64] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] - stp d12, d13, [x29, #112] - stp d14, d15, [x29, #128] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] + stp D12, D13, [x29, #112] + stp D14, D15, [x29, #128] adrp x20, L_AES_ARM64_NEON_te add x20, x20, L_AES_ARM64_NEON_te adrp x21, L_AES_ARM64_NEON_td @@ -49178,138 +49178,138 @@ AES_XTS_decrypt_NEON PROC add x22, x22, L_AES_ARM64_NEON_shift_rows_shuffle adrp x23, L_AES_ARM64_NEON_shift_rows_invshuffle add x23, x23, L_AES_ARM64_NEON_shift_rows_invshuffle - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x20], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x20], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x20], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x20] - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 - ld1 {v3.2d}, [x22] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x20], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x20], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x20], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x20] + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 + ld1 {V3.2D}, [x22] mov x17, #0x87 ands w19, w2, #15 cset w16, ne sub w2, w2, w16, lsl 4 - ld1 {v2.2d}, [x3] - ld1 {v4.2d}, [x5] - rev32 v2.16b, v2.16b + ld1 {V2.2D}, [x3] + ld1 {V4.2D}, [x5] + rev32 V2.16B, V2.16B add x25, x5, #16 ; Round: 0 - XOR in key schedule - eor v2.16b, v2.16b, v4.16b + eor V2.16B, V2.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_tweak - eor v8.16b, v2.16b, v12.16b - eor v9.16b, v2.16b, v13.16b - eor v10.16b, v2.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v2.2d}, [x25], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V2.16B, V12.16B + eor V9.16B, V2.16B, V13.16B + eor V10.16B, V2.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V2.2D}, [x25], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v2.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v2.16b, v2.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v2.16b, v2.16b, v9.16b - tbl v2.16b, {v2.16b}, v3.16b - ld1 {v4.2d}, [x25], #16 - sshr v10.16b, v2.16b, #7 - shl v9.16b, v2.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v2.8h - eor v11.16b, v10.16b, v2.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v2.4s, #24 - shl v8.4s, v11.4s, #8 + eor V10.16B, V10.16B, V2.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V2.16B, V2.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V2.16B, V2.16B, V9.16B + tbl V2.16B, {V2.16B}, V3.16B + ld1 {V4.2D}, [x25], #16 + sshr V10.16B, V2.16B, #7 + shl V9.16B, V2.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V2.8H + eor V11.16B, V10.16B, V2.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V2.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v4.16b - sri v9.4s, v2.4s, #8 - sri v8.4s, v11.4s, #24 - eor v2.16b, v10.16b, v9.16b - eor v2.16b, v2.16b, v8.16b + eor V10.16B, V10.16B, V4.16B + sri V9.4S, V2.4S, #8 + sri V8.4S, V11.4S, #24 + eor V2.16B, V10.16B, V9.16B + eor V2.16B, V2.16B, V8.16B subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_tweak - eor v8.16b, v2.16b, v12.16b - eor v9.16b, v2.16b, v13.16b - eor v10.16b, v2.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - ld1 {v2.2d}, [x25], #16 - sshr v10.16b, v4.16b, #7 - shl v9.16b, v4.16b, #1 - and v10.16b, v10.16b, v15.16b - eor v10.16b, v10.16b, v9.16b - rev32 v8.8h, v4.8h - eor v11.16b, v10.16b, v4.16b - eor v10.16b, v10.16b, v8.16b - shl v9.4s, v4.4s, #24 - shl v8.4s, v11.4s, #8 + eor V8.16B, V2.16B, V12.16B + eor V9.16B, V2.16B, V13.16B + eor V10.16B, V2.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + ld1 {V2.2D}, [x25], #16 + sshr V10.16B, V4.16B, #7 + shl V9.16B, V4.16B, #1 + and V10.16B, V10.16B, V15.16B + eor V10.16B, V10.16B, V9.16B + rev32 V8.8H, V4.8H + eor V11.16B, V10.16B, V4.16B + eor V10.16B, V10.16B, V8.16B + shl V9.4S, V4.4S, #24 + shl V8.4S, V11.4S, #8 ; XOR in Key Schedule - eor v10.16b, v10.16b, v2.16b - sri v9.4s, v4.4s, #8 - sri v8.4s, v11.4s, #24 - eor v4.16b, v10.16b, v9.16b - eor v4.16b, v4.16b, v8.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v2.16b, v2.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v2.16b, v2.16b, v9.16b - tbl v2.16b, {v2.16b}, v3.16b - ld1 {v4.2d}, [x25], #16 + eor V10.16B, V10.16B, V2.16B + sri V9.4S, V4.4S, #8 + sri V8.4S, V11.4S, #24 + eor V4.16B, V10.16B, V9.16B + eor V4.16B, V4.16B, V8.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V2.16B, V2.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V2.16B, V2.16B, V9.16B + tbl V2.16B, {V2.16B}, V3.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v2.16b, v2.16b, v4.16b - rev32 v2.16b, v2.16b - mov x8, v2.d[0] - mov x9, v2.d[1] - ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x21], #0x40 - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x21], #0x40 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x21], #0x40 - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] - ld1 {v3.2d}, [x23] + eor V2.16B, V2.16B, V4.16B + rev32 V2.16B, V2.16B + mov x8, V2.D[0] + mov x9, V2.D[1] + ld1 {V16.16B, V17.16B, V18.16B, V19.16B}, [x21], #0x40 + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x21], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x21], #0x40 + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x21] + ld1 {V3.2D}, [x23] cmp w2, #0x40 blt L_AES_XTS_decrypt_NEON_start_2 L_AES_XTS_decrypt_NEON_loop_4 mov x25, x4 - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld1 {v4.16b}, [x25], #16 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld1 {V4.16B}, [x25], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 @@ -49319,1045 +49319,1045 @@ L_AES_XTS_decrypt_NEON_loop_4 and x16, x17, x13, asr 63 extr x15, x13, x12, #63 eor x14, x16, x12, lsl 1 - mov v8.d[0], x8 - mov v8.d[1], x9 - mov v9.d[0], x10 - mov v9.d[1], x11 - mov v10.d[0], x12 - mov v10.d[1], x13 - mov v11.d[0], x14 - mov v11.d[1], x15 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + mov V8.D[0], x8 + mov V8.D[1], x9 + mov V9.D[0], x10 + mov V9.D[1], x11 + mov V10.D[0], x12 + mov V10.D[1], x13 + mov V11.D[0], x14 + mov V11.D[1], x15 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x23] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - movi v28.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - ushr v14.16b, v6.16b, #6 - ushr v15.16b, v7.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - shl v2.16b, v6.16b, #2 - shl v3.16b, v7.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - ushr v2.16b, v6.16b, #5 - ushr v3.16b, v7.16b, #5 - pmul v0.16b, v0.16b, v28.16b - pmul v1.16b, v1.16b, v28.16b - pmul v2.16b, v2.16b, v28.16b - pmul v3.16b, v3.16b, v28.16b - shl v28.16b, v4.16b, #3 - shl v29.16b, v5.16b, #3 - shl v30.16b, v6.16b, #3 - shl v31.16b, v7.16b, #3 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - eor v28.16b, v8.16b, v0.16b - eor v29.16b, v9.16b, v1.16b - eor v30.16b, v10.16b, v2.16b - eor v31.16b, v11.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v10.16b, v14.16b, v2.16b - eor v11.16b, v15.16b, v3.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v4.16b - eor v29.16b, v29.16b, v5.16b - eor v30.16b, v30.16b, v6.16b - eor v31.16b, v31.16b, v7.16b - shl v4.4s, v28.4s, #8 - shl v5.4s, v29.4s, #8 - shl v6.4s, v30.4s, #8 - shl v7.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v4.4s, v28.4s, #24 - sri v5.4s, v29.4s, #24 - sri v6.4s, v30.4s, #24 - sri v7.4s, v31.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - shl v28.4s, v0.4s, #24 - shl v29.4s, v1.4s, #24 - shl v30.4s, v2.4s, #24 - shl v31.4s, v3.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - sri v28.4s, v0.4s, #8 - sri v29.4s, v1.4s, #8 - sri v30.4s, v2.4s, #8 - sri v31.4s, v3.4s, #8 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x23] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + movi V28.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + ushr V14.16B, V6.16B, #6 + ushr V15.16B, V7.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + shl V2.16B, V6.16B, #2 + shl V3.16B, V7.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + ushr V2.16B, V6.16B, #5 + ushr V3.16B, V7.16B, #5 + pmul V0.16B, V0.16B, V28.16B + pmul V1.16B, V1.16B, V28.16B + pmul V2.16B, V2.16B, V28.16B + pmul V3.16B, V3.16B, V28.16B + shl V28.16B, V4.16B, #3 + shl V29.16B, V5.16B, #3 + shl V30.16B, V6.16B, #3 + shl V31.16B, V7.16B, #3 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + eor V28.16B, V8.16B, V0.16B + eor V29.16B, V9.16B, V1.16B + eor V30.16B, V10.16B, V2.16B + eor V31.16B, V11.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V10.16B, V14.16B, V2.16B + eor V11.16B, V15.16B, V3.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V4.16B + eor V29.16B, V29.16B, V5.16B + eor V30.16B, V30.16B, V6.16B + eor V31.16B, V31.16B, V7.16B + shl V4.4S, V28.4S, #8 + shl V5.4S, V29.4S, #8 + shl V6.4S, V30.4S, #8 + shl V7.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V4.4S, V28.4S, #24 + sri V5.4S, V29.4S, #24 + sri V6.4S, V30.4S, #24 + sri V7.4S, V31.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B + shl V28.4S, V0.4S, #24 + shl V29.4S, V1.4S, #24 + shl V30.4S, V2.4S, #24 + shl V31.4S, V3.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + sri V28.4S, V0.4S, #8 + sri V29.4S, V1.4S, #8 + sri V30.4S, V2.4S, #8 + sri V31.4S, V3.4S, #8 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x21] ; XOR in Key Schedule - ld1 {v0.2d}, [x25], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b + ld1 {V0.2D}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x23] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b - movi v28.16b, #27 - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - sshr v10.16b, v2.16b, #7 - sshr v11.16b, v3.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - shl v14.16b, v2.16b, #1 - shl v15.16b, v3.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v0.16b, #6 - ushr v13.16b, v1.16b, #6 - ushr v14.16b, v2.16b, #6 - ushr v15.16b, v3.16b, #6 - shl v4.16b, v0.16b, #2 - shl v5.16b, v1.16b, #2 - shl v6.16b, v2.16b, #2 - shl v7.16b, v3.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - eor v14.16b, v14.16b, v6.16b - eor v15.16b, v15.16b, v7.16b - ushr v4.16b, v0.16b, #5 - ushr v5.16b, v1.16b, #5 - ushr v6.16b, v2.16b, #5 - ushr v7.16b, v3.16b, #5 - pmul v4.16b, v4.16b, v28.16b - pmul v5.16b, v5.16b, v28.16b - pmul v6.16b, v6.16b, v28.16b - pmul v7.16b, v7.16b, v28.16b - shl v28.16b, v0.16b, #3 - shl v29.16b, v1.16b, #3 - shl v30.16b, v2.16b, #3 - shl v31.16b, v3.16b, #3 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - eor v28.16b, v8.16b, v4.16b - eor v29.16b, v9.16b, v5.16b - eor v30.16b, v10.16b, v6.16b - eor v31.16b, v11.16b, v7.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v6.16b, v6.16b, v2.16b - eor v7.16b, v7.16b, v3.16b - eor v8.16b, v12.16b, v4.16b - eor v9.16b, v13.16b, v5.16b - eor v10.16b, v14.16b, v6.16b - eor v11.16b, v15.16b, v7.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v0.16b - eor v29.16b, v29.16b, v1.16b - eor v30.16b, v30.16b, v2.16b - eor v31.16b, v31.16b, v3.16b - shl v0.4s, v28.4s, #8 - shl v1.4s, v29.4s, #8 - shl v2.4s, v30.4s, #8 - shl v3.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v0.4s, v28.4s, #24 - sri v1.4s, v29.4s, #24 - sri v2.4s, v30.4s, #24 - sri v3.4s, v31.4s, #24 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - eor v2.16b, v2.16b, v14.16b - eor v3.16b, v3.16b, v15.16b - shl v28.4s, v4.4s, #24 - shl v29.4s, v5.4s, #24 - shl v30.4s, v6.4s, #24 - shl v31.4s, v7.4s, #24 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - sri v28.4s, v4.4s, #8 - sri v29.4s, v5.4s, #8 - sri v30.4s, v6.4s, #8 - sri v31.4s, v7.4s, #8 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x23] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B + movi V28.16B, #27 + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + sshr V10.16B, V2.16B, #7 + sshr V11.16B, V3.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + shl V14.16B, V2.16B, #1 + shl V15.16B, V3.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V0.16B, #6 + ushr V13.16B, V1.16B, #6 + ushr V14.16B, V2.16B, #6 + ushr V15.16B, V3.16B, #6 + shl V4.16B, V0.16B, #2 + shl V5.16B, V1.16B, #2 + shl V6.16B, V2.16B, #2 + shl V7.16B, V3.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + eor V14.16B, V14.16B, V6.16B + eor V15.16B, V15.16B, V7.16B + ushr V4.16B, V0.16B, #5 + ushr V5.16B, V1.16B, #5 + ushr V6.16B, V2.16B, #5 + ushr V7.16B, V3.16B, #5 + pmul V4.16B, V4.16B, V28.16B + pmul V5.16B, V5.16B, V28.16B + pmul V6.16B, V6.16B, V28.16B + pmul V7.16B, V7.16B, V28.16B + shl V28.16B, V0.16B, #3 + shl V29.16B, V1.16B, #3 + shl V30.16B, V2.16B, #3 + shl V31.16B, V3.16B, #3 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + eor V28.16B, V8.16B, V4.16B + eor V29.16B, V9.16B, V5.16B + eor V30.16B, V10.16B, V6.16B + eor V31.16B, V11.16B, V7.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V6.16B, V6.16B, V2.16B + eor V7.16B, V7.16B, V3.16B + eor V8.16B, V12.16B, V4.16B + eor V9.16B, V13.16B, V5.16B + eor V10.16B, V14.16B, V6.16B + eor V11.16B, V15.16B, V7.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V0.16B + eor V29.16B, V29.16B, V1.16B + eor V30.16B, V30.16B, V2.16B + eor V31.16B, V31.16B, V3.16B + shl V0.4S, V28.4S, #8 + shl V1.4S, V29.4S, #8 + shl V2.4S, V30.4S, #8 + shl V3.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V0.4S, V28.4S, #24 + sri V1.4S, V29.4S, #24 + sri V2.4S, V30.4S, #24 + sri V3.4S, V31.4S, #24 + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + eor V2.16B, V2.16B, V14.16B + eor V3.16B, V3.16B, V15.16B + shl V28.4S, V4.4S, #24 + shl V29.4S, V5.4S, #24 + shl V30.4S, V6.4S, #24 + shl V31.4S, V7.4S, #24 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + sri V28.4S, V4.4S, #8 + sri V29.4S, V5.4S, #8 + sri V30.4S, V6.4S, #8 + sri V31.4S, V7.4S, #8 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x21] ; XOR in Key Schedule - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_4 - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v6.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v2.16b - tbl v7.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v3.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - eor v10.16b, v2.16b, v12.16b - eor v11.16b, v3.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v13.16b - eor v9.16b, v1.16b, v13.16b - eor v10.16b, v2.16b, v13.16b - eor v11.16b, v3.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - eor v10.16b, v2.16b, v14.16b - eor v11.16b, v3.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - orr v6.16b, v6.16b, v10.16b - orr v7.16b, v7.16b, v11.16b - ld1 {v0.16b}, [x23] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - tbl v6.16b, {v6.16b}, v0.16b - tbl v7.16b, {v7.16b}, v0.16b - movi v28.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - sshr v10.16b, v6.16b, #7 - sshr v11.16b, v7.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - shl v14.16b, v6.16b, #1 - shl v15.16b, v7.16b, #1 - and v8.16b, v8.16b, v28.16b - and v9.16b, v9.16b, v28.16b - and v10.16b, v10.16b, v28.16b - and v11.16b, v11.16b, v28.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - eor v10.16b, v10.16b, v14.16b - eor v11.16b, v11.16b, v15.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - ushr v14.16b, v6.16b, #6 - ushr v15.16b, v7.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - shl v2.16b, v6.16b, #2 - shl v3.16b, v7.16b, #2 - pmul v12.16b, v12.16b, v28.16b - pmul v13.16b, v13.16b, v28.16b - pmul v14.16b, v14.16b, v28.16b - pmul v15.16b, v15.16b, v28.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - ushr v2.16b, v6.16b, #5 - ushr v3.16b, v7.16b, #5 - pmul v0.16b, v0.16b, v28.16b - pmul v1.16b, v1.16b, v28.16b - pmul v2.16b, v2.16b, v28.16b - pmul v3.16b, v3.16b, v28.16b - shl v28.16b, v4.16b, #3 - shl v29.16b, v5.16b, #3 - shl v30.16b, v6.16b, #3 - shl v31.16b, v7.16b, #3 - eor v0.16b, v0.16b, v28.16b - eor v1.16b, v1.16b, v29.16b - eor v2.16b, v2.16b, v30.16b - eor v3.16b, v3.16b, v31.16b - eor v28.16b, v8.16b, v0.16b - eor v29.16b, v9.16b, v1.16b - eor v30.16b, v10.16b, v2.16b - eor v31.16b, v11.16b, v3.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v10.16b, v14.16b, v2.16b - eor v11.16b, v15.16b, v3.16b - eor v12.16b, v12.16b, v28.16b - eor v13.16b, v13.16b, v29.16b - eor v14.16b, v14.16b, v30.16b - eor v15.16b, v15.16b, v31.16b - eor v28.16b, v28.16b, v4.16b - eor v29.16b, v29.16b, v5.16b - eor v30.16b, v30.16b, v6.16b - eor v31.16b, v31.16b, v7.16b - shl v4.4s, v28.4s, #8 - shl v5.4s, v29.4s, #8 - shl v6.4s, v30.4s, #8 - shl v7.4s, v31.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - rev32 v10.8h, v10.8h - rev32 v11.8h, v11.8h - sri v4.4s, v28.4s, #24 - sri v5.4s, v29.4s, #24 - sri v6.4s, v30.4s, #24 - sri v7.4s, v31.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - shl v28.4s, v0.4s, #24 - shl v29.4s, v1.4s, #24 - shl v30.4s, v2.4s, #24 - shl v31.4s, v3.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - eor v6.16b, v6.16b, v10.16b - eor v7.16b, v7.16b, v11.16b - sri v28.4s, v0.4s, #8 - sri v29.4s, v1.4s, #8 - sri v30.4s, v2.4s, #8 - sri v31.4s, v3.4s, #8 - eor v4.16b, v4.16b, v28.16b - eor v5.16b, v5.16b, v29.16b - eor v6.16b, v6.16b, v30.16b - eor v7.16b, v7.16b, v31.16b - ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x21] + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V6.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V2.16B + tbl V7.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V3.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + eor V10.16B, V2.16B, V12.16B + eor V11.16B, V3.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V13.16B + eor V9.16B, V1.16B, V13.16B + eor V10.16B, V2.16B, V13.16B + eor V11.16B, V3.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + eor V10.16B, V2.16B, V14.16B + eor V11.16B, V3.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + orr V6.16B, V6.16B, V10.16B + orr V7.16B, V7.16B, V11.16B + ld1 {V0.16B}, [x23] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + tbl V6.16B, {V6.16B}, V0.16B + tbl V7.16B, {V7.16B}, V0.16B + movi V28.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + sshr V10.16B, V6.16B, #7 + sshr V11.16B, V7.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + shl V14.16B, V6.16B, #1 + shl V15.16B, V7.16B, #1 + and V8.16B, V8.16B, V28.16B + and V9.16B, V9.16B, V28.16B + and V10.16B, V10.16B, V28.16B + and V11.16B, V11.16B, V28.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + eor V10.16B, V10.16B, V14.16B + eor V11.16B, V11.16B, V15.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + ushr V14.16B, V6.16B, #6 + ushr V15.16B, V7.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + shl V2.16B, V6.16B, #2 + shl V3.16B, V7.16B, #2 + pmul V12.16B, V12.16B, V28.16B + pmul V13.16B, V13.16B, V28.16B + pmul V14.16B, V14.16B, V28.16B + pmul V15.16B, V15.16B, V28.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + ushr V2.16B, V6.16B, #5 + ushr V3.16B, V7.16B, #5 + pmul V0.16B, V0.16B, V28.16B + pmul V1.16B, V1.16B, V28.16B + pmul V2.16B, V2.16B, V28.16B + pmul V3.16B, V3.16B, V28.16B + shl V28.16B, V4.16B, #3 + shl V29.16B, V5.16B, #3 + shl V30.16B, V6.16B, #3 + shl V31.16B, V7.16B, #3 + eor V0.16B, V0.16B, V28.16B + eor V1.16B, V1.16B, V29.16B + eor V2.16B, V2.16B, V30.16B + eor V3.16B, V3.16B, V31.16B + eor V28.16B, V8.16B, V0.16B + eor V29.16B, V9.16B, V1.16B + eor V30.16B, V10.16B, V2.16B + eor V31.16B, V11.16B, V3.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V10.16B, V14.16B, V2.16B + eor V11.16B, V15.16B, V3.16B + eor V12.16B, V12.16B, V28.16B + eor V13.16B, V13.16B, V29.16B + eor V14.16B, V14.16B, V30.16B + eor V15.16B, V15.16B, V31.16B + eor V28.16B, V28.16B, V4.16B + eor V29.16B, V29.16B, V5.16B + eor V30.16B, V30.16B, V6.16B + eor V31.16B, V31.16B, V7.16B + shl V4.4S, V28.4S, #8 + shl V5.4S, V29.4S, #8 + shl V6.4S, V30.4S, #8 + shl V7.4S, V31.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + rev32 V10.8H, V10.8H + rev32 V11.8H, V11.8H + sri V4.4S, V28.4S, #24 + sri V5.4S, V29.4S, #24 + sri V6.4S, V30.4S, #24 + sri V7.4S, V31.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + eor V6.16B, V6.16B, V14.16B + eor V7.16B, V7.16B, V15.16B + shl V28.4S, V0.4S, #24 + shl V29.4S, V1.4S, #24 + shl V30.4S, V2.4S, #24 + shl V31.4S, V3.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + eor V6.16B, V6.16B, V10.16B + eor V7.16B, V7.16B, V11.16B + sri V28.4S, V0.4S, #8 + sri V29.4S, V1.4S, #8 + sri V30.4S, V2.4S, #8 + sri V31.4S, V3.4S, #8 + eor V4.16B, V4.16B, V28.16B + eor V5.16B, V5.16B, V29.16B + eor V6.16B, V6.16B, V30.16B + eor V7.16B, V7.16B, V31.16B + ld1 {V28.16B, V29.16B, V30.16B, V31.16B}, [x21] ; XOR in Key Schedule - ld1 {v0.2d}, [x25], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b - eor v6.16b, v6.16b, v0.16b - eor v7.16b, v7.16b, v0.16b + ld1 {V0.2D}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B + eor V6.16B, V6.16B, V0.16B + eor V7.16B, V7.16B, V0.16B ; Round Done - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v2.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v6.16b - tbl v3.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v7.16b - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - eor v10.16b, v6.16b, v12.16b - eor v11.16b, v7.16b, v12.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - tbl v10.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v10.16b - tbl v11.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v13.16b - eor v9.16b, v5.16b, v13.16b - eor v10.16b, v6.16b, v13.16b - eor v11.16b, v7.16b, v13.16b - tbl v8.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - eor v10.16b, v6.16b, v14.16b - eor v11.16b, v7.16b, v14.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - tbl v11.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - orr v2.16b, v2.16b, v10.16b - orr v3.16b, v3.16b, v11.16b - ld1 {v4.16b}, [x23] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - tbl v2.16b, {v2.16b}, v4.16b - tbl v3.16b, {v3.16b}, v4.16b + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V2.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V6.16B + tbl V3.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V7.16B + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + eor V10.16B, V6.16B, V12.16B + eor V11.16B, V7.16B, V12.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + tbl V10.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V10.16B + tbl V11.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V13.16B + eor V9.16B, V5.16B, V13.16B + eor V10.16B, V6.16B, V13.16B + eor V11.16B, V7.16B, V13.16B + tbl V8.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + eor V10.16B, V6.16B, V14.16B + eor V11.16B, V7.16B, V14.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + tbl V11.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + orr V2.16B, V2.16B, V10.16B + orr V3.16B, V3.16B, V11.16B + ld1 {V4.16B}, [x23] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + tbl V2.16B, {V2.16B}, V4.16B + tbl V3.16B, {V3.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b - eor v2.16b, v2.16b, v4.16b - eor v3.16b, v3.16b, v4.16b + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B + eor V2.16B, V2.16B, V4.16B + eor V3.16B, V3.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - rev32 v2.16b, v2.16b - rev32 v3.16b, v3.16b - mov v8.d[0], x8 - mov v8.d[1], x9 - mov v9.d[0], x10 - mov v9.d[1], x11 - mov v10.d[0], x12 - mov v10.d[1], x13 - mov v11.d[0], x14 - mov v11.d[1], x15 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + rev32 V2.16B, V2.16B + rev32 V3.16B, V3.16B + mov V8.D[0], x8 + mov V8.D[1], x9 + mov V9.D[0], x10 + mov V9.D[1], x11 + mov V10.D[0], x12 + mov V10.D[1], x13 + mov V11.D[0], x14 + mov V11.D[1], x15 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + eor V2.16B, V2.16B, V10.16B + eor V3.16B, V3.16B, V11.16B + st1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 and x16, x17, x15, asr 63 extr x9, x15, x14, #63 eor x8, x16, x14, lsl 1 sub w2, w2, #0x40 cmp w2, #0x40 bge L_AES_XTS_decrypt_NEON_loop_4 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - movi v15.16b, #27 + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + movi V15.16B, #27 L_AES_XTS_decrypt_NEON_start_2 cmp w2, #32 blt L_AES_XTS_decrypt_NEON_start_1 mov x25, x4 - ld1 {v0.16b, v1.16b}, [x0], #32 - ld1 {v4.16b}, [x25], #16 + ld1 {V0.16B, V1.16B}, [x0], #32 + ld1 {V4.16B}, [x25], #16 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 and x16, x17, x11, asr 63 extr x13, x11, x10, #63 eor x12, x16, x10, lsl 1 - mov v2.d[0], x8 - mov v2.d[1], x9 - mov v3.d[0], x10 - mov v3.d[1], x11 - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v3.16b - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + mov V2.D[0], x8 + mov V2.D[1], x9 + mov V3.D[0], x10 + mov V3.D[1], x11 + eor V0.16B, V0.16B, V2.16B + eor V1.16B, V1.16B, V3.16B + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x23] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - movi v10.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - pmul v0.16b, v0.16b, v10.16b - pmul v1.16b, v1.16b, v10.16b - shl v10.16b, v4.16b, #3 - shl v11.16b, v5.16b, #3 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b - eor v10.16b, v8.16b, v0.16b - eor v11.16b, v9.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v5.16b - shl v4.4s, v10.4s, #8 - shl v5.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v4.4s, v10.4s, #24 - sri v5.4s, v11.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - shl v10.4s, v0.4s, #24 - shl v11.4s, v1.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - sri v10.4s, v0.4s, #8 - sri v11.4s, v1.4s, #8 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x23] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + movi V10.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + pmul V0.16B, V0.16B, V10.16B + pmul V1.16B, V1.16B, V10.16B + shl V10.16B, V4.16B, #3 + shl V11.16B, V5.16B, #3 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B + eor V10.16B, V8.16B, V0.16B + eor V11.16B, V9.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V5.16B + shl V4.4S, V10.4S, #8 + shl V5.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V4.4S, V10.4S, #24 + sri V5.4S, V11.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + shl V10.4S, V0.4S, #24 + shl V11.4S, V1.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + sri V10.4S, V0.4S, #8 + sri V11.4S, V1.4S, #8 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x25], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b + ld1 {V0.2D}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x23] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b - movi v10.16b, #27 - sshr v8.16b, v0.16b, #7 - sshr v9.16b, v1.16b, #7 - shl v12.16b, v0.16b, #1 - shl v13.16b, v1.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v0.16b, #6 - ushr v13.16b, v1.16b, #6 - shl v4.16b, v0.16b, #2 - shl v5.16b, v1.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v4.16b - eor v13.16b, v13.16b, v5.16b - ushr v4.16b, v0.16b, #5 - ushr v5.16b, v1.16b, #5 - pmul v4.16b, v4.16b, v10.16b - pmul v5.16b, v5.16b, v10.16b - shl v10.16b, v0.16b, #3 - shl v11.16b, v1.16b, #3 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b - eor v10.16b, v8.16b, v4.16b - eor v11.16b, v9.16b, v5.16b - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v1.16b - eor v8.16b, v12.16b, v4.16b - eor v9.16b, v13.16b, v5.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v0.16b - eor v11.16b, v11.16b, v1.16b - shl v0.4s, v10.4s, #8 - shl v1.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v0.4s, v10.4s, #24 - sri v1.4s, v11.4s, #24 - eor v0.16b, v0.16b, v12.16b - eor v1.16b, v1.16b, v13.16b - shl v10.4s, v4.4s, #24 - shl v11.4s, v5.4s, #24 - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - sri v10.4s, v4.4s, #8 - sri v11.4s, v5.4s, #8 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x23] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B + movi V10.16B, #27 + sshr V8.16B, V0.16B, #7 + sshr V9.16B, V1.16B, #7 + shl V12.16B, V0.16B, #1 + shl V13.16B, V1.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V0.16B, #6 + ushr V13.16B, V1.16B, #6 + shl V4.16B, V0.16B, #2 + shl V5.16B, V1.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V4.16B + eor V13.16B, V13.16B, V5.16B + ushr V4.16B, V0.16B, #5 + ushr V5.16B, V1.16B, #5 + pmul V4.16B, V4.16B, V10.16B + pmul V5.16B, V5.16B, V10.16B + shl V10.16B, V0.16B, #3 + shl V11.16B, V1.16B, #3 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B + eor V10.16B, V8.16B, V4.16B + eor V11.16B, V9.16B, V5.16B + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V1.16B + eor V8.16B, V12.16B, V4.16B + eor V9.16B, V13.16B, V5.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V0.16B + eor V11.16B, V11.16B, V1.16B + shl V0.4S, V10.4S, #8 + shl V1.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V0.4S, V10.4S, #24 + sri V1.4S, V11.4S, #24 + eor V0.16B, V0.16B, V12.16B + eor V1.16B, V1.16B, V13.16B + shl V10.4S, V4.4S, #24 + shl V11.4S, V5.4S, #24 + eor V0.16B, V0.16B, V8.16B + eor V1.16B, V1.16B, V9.16B + sri V10.4S, V4.4S, #8 + sri V11.4S, V5.4S, #8 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_2 - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v1.16b, v12.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v5.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v1.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v0.16b, v13.16b - eor v11.16b, v1.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - eor v8.16b, v0.16b, v14.16b - eor v9.16b, v1.16b, v14.16b - orr v4.16b, v4.16b, v10.16b - orr v5.16b, v5.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v4.16b, v4.16b, v8.16b - orr v5.16b, v5.16b, v9.16b - ld1 {v0.16b}, [x23] - tbl v4.16b, {v4.16b}, v0.16b - tbl v5.16b, {v5.16b}, v0.16b - movi v10.16b, #27 - sshr v8.16b, v4.16b, #7 - sshr v9.16b, v5.16b, #7 - shl v12.16b, v4.16b, #1 - shl v13.16b, v5.16b, #1 - and v8.16b, v8.16b, v10.16b - and v9.16b, v9.16b, v10.16b - eor v8.16b, v8.16b, v12.16b - eor v9.16b, v9.16b, v13.16b - ushr v12.16b, v4.16b, #6 - ushr v13.16b, v5.16b, #6 - shl v0.16b, v4.16b, #2 - shl v1.16b, v5.16b, #2 - pmul v12.16b, v12.16b, v10.16b - pmul v13.16b, v13.16b, v10.16b - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - ushr v0.16b, v4.16b, #5 - ushr v1.16b, v5.16b, #5 - pmul v0.16b, v0.16b, v10.16b - pmul v1.16b, v1.16b, v10.16b - shl v10.16b, v4.16b, #3 - shl v11.16b, v5.16b, #3 - eor v0.16b, v0.16b, v10.16b - eor v1.16b, v1.16b, v11.16b - eor v10.16b, v8.16b, v0.16b - eor v11.16b, v9.16b, v1.16b - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v8.16b, v12.16b, v0.16b - eor v9.16b, v13.16b, v1.16b - eor v12.16b, v12.16b, v10.16b - eor v13.16b, v13.16b, v11.16b - eor v10.16b, v10.16b, v4.16b - eor v11.16b, v11.16b, v5.16b - shl v4.4s, v10.4s, #8 - shl v5.4s, v11.4s, #8 - rev32 v8.8h, v8.8h - rev32 v9.8h, v9.8h - sri v4.4s, v10.4s, #24 - sri v5.4s, v11.4s, #24 - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - shl v10.4s, v0.4s, #24 - shl v11.4s, v1.4s, #24 - eor v4.16b, v4.16b, v8.16b - eor v5.16b, v5.16b, v9.16b - sri v10.4s, v0.4s, #8 - sri v11.4s, v1.4s, #8 - eor v4.16b, v4.16b, v10.16b - eor v5.16b, v5.16b, v11.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V1.16B, V12.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V5.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V1.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V0.16B, V13.16B + eor V11.16B, V1.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + eor V8.16B, V0.16B, V14.16B + eor V9.16B, V1.16B, V14.16B + orr V4.16B, V4.16B, V10.16B + orr V5.16B, V5.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V4.16B, V4.16B, V8.16B + orr V5.16B, V5.16B, V9.16B + ld1 {V0.16B}, [x23] + tbl V4.16B, {V4.16B}, V0.16B + tbl V5.16B, {V5.16B}, V0.16B + movi V10.16B, #27 + sshr V8.16B, V4.16B, #7 + sshr V9.16B, V5.16B, #7 + shl V12.16B, V4.16B, #1 + shl V13.16B, V5.16B, #1 + and V8.16B, V8.16B, V10.16B + and V9.16B, V9.16B, V10.16B + eor V8.16B, V8.16B, V12.16B + eor V9.16B, V9.16B, V13.16B + ushr V12.16B, V4.16B, #6 + ushr V13.16B, V5.16B, #6 + shl V0.16B, V4.16B, #2 + shl V1.16B, V5.16B, #2 + pmul V12.16B, V12.16B, V10.16B + pmul V13.16B, V13.16B, V10.16B + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + ushr V0.16B, V4.16B, #5 + ushr V1.16B, V5.16B, #5 + pmul V0.16B, V0.16B, V10.16B + pmul V1.16B, V1.16B, V10.16B + shl V10.16B, V4.16B, #3 + shl V11.16B, V5.16B, #3 + eor V0.16B, V0.16B, V10.16B + eor V1.16B, V1.16B, V11.16B + eor V10.16B, V8.16B, V0.16B + eor V11.16B, V9.16B, V1.16B + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V8.16B, V12.16B, V0.16B + eor V9.16B, V13.16B, V1.16B + eor V12.16B, V12.16B, V10.16B + eor V13.16B, V13.16B, V11.16B + eor V10.16B, V10.16B, V4.16B + eor V11.16B, V11.16B, V5.16B + shl V4.4S, V10.4S, #8 + shl V5.4S, V11.4S, #8 + rev32 V8.8H, V8.8H + rev32 V9.8H, V9.8H + sri V4.4S, V10.4S, #24 + sri V5.4S, V11.4S, #24 + eor V4.16B, V4.16B, V12.16B + eor V5.16B, V5.16B, V13.16B + shl V10.4S, V0.4S, #24 + shl V11.4S, V1.4S, #24 + eor V4.16B, V4.16B, V8.16B + eor V5.16B, V5.16B, V9.16B + sri V10.4S, V0.4S, #8 + sri V11.4S, V1.4S, #8 + eor V4.16B, V4.16B, V10.16B + eor V5.16B, V5.16B, V11.16B ; XOR in Key Schedule - ld1 {v0.2d}, [x25], #16 - eor v4.16b, v4.16b, v0.16b - eor v5.16b, v5.16b, v0.16b + ld1 {V0.2D}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V5.16B, V5.16B, V0.16B ; Round Done - movi v12.16b, #0x40 - movi v13.16b, #0x80 - movi v14.16b, #0xc0 - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v5.16b, v12.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v1.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v5.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v9.16b - eor v10.16b, v4.16b, v13.16b - eor v11.16b, v5.16b, v13.16b - tbl v10.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v10.16b - tbl v11.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v11.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - eor v8.16b, v4.16b, v14.16b - eor v9.16b, v5.16b, v14.16b - orr v0.16b, v0.16b, v10.16b - orr v1.16b, v1.16b, v11.16b - tbl v8.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v8.16b - tbl v9.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v9.16b - orr v0.16b, v0.16b, v8.16b - orr v1.16b, v1.16b, v9.16b - ld1 {v4.16b}, [x23] - tbl v0.16b, {v0.16b}, v4.16b - tbl v1.16b, {v1.16b}, v4.16b + movi V12.16B, #0x40 + movi V13.16B, #0x80 + movi V14.16B, #0xc0 + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V5.16B, V12.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V1.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V5.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V9.16B + eor V10.16B, V4.16B, V13.16B + eor V11.16B, V5.16B, V13.16B + tbl V10.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V10.16B + tbl V11.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V11.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + eor V8.16B, V4.16B, V14.16B + eor V9.16B, V5.16B, V14.16B + orr V0.16B, V0.16B, V10.16B + orr V1.16B, V1.16B, V11.16B + tbl V8.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V8.16B + tbl V9.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V9.16B + orr V0.16B, V0.16B, V8.16B + orr V1.16B, V1.16B, V9.16B + ld1 {V4.16B}, [x23] + tbl V0.16B, {V0.16B}, V4.16B + tbl V1.16B, {V1.16B}, V4.16B ; XOR in Key Schedule - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v4.16b + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V4.16B ; Round Done - rev32 v0.16b, v0.16b - rev32 v1.16b, v1.16b - eor v0.16b, v0.16b, v2.16b - eor v1.16b, v1.16b, v3.16b - st1 {v0.16b, v1.16b}, [x1], #32 + rev32 V0.16B, V0.16B + rev32 V1.16B, V1.16B + eor V0.16B, V0.16B, V2.16B + eor V1.16B, V1.16B, V3.16B + st1 {V0.16B, V1.16B}, [x1], #32 and x16, x17, x11, asr 63 extr x9, x11, x10, #63 eor x8, x16, x10, lsl 1 sub w2, w2, #32 L_AES_XTS_decrypt_NEON_start_1 - ld1 {v3.2d}, [x23] - mov v2.d[0], x8 - mov v2.d[1], x9 + ld1 {V3.2D}, [x23] + mov V2.D[0], x8 + mov V2.D[1], x9 cmp w2, #16 blt L_AES_XTS_decrypt_NEON_start_partial mov x25, x4 - ld1 {v0.16b}, [x0], #16 - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v2.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v4.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V2.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - sshr v10.16b, v0.16b, #7 - ushr v11.16b, v0.16b, #6 - ushr v8.16b, v0.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v0.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v0.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v0.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v0.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v0.16b - shl v0.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v0.4s, v9.4s, #24 - eor v0.16b, v0.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v0.16b, v0.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v0.16b, v0.16b, v9.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + sshr V10.16B, V0.16B, #7 + ushr V11.16B, V0.16B, #6 + ushr V8.16B, V0.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V0.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V0.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V0.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V0.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V0.16B + shl V0.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V0.4S, V9.4S, #24 + eor V0.16B, V0.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V0.16B, V0.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V0.16B, V0.16B, V9.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v2.16b - st1 {v0.16b}, [x1], #16 + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V2.16B + st1 {V0.16B}, [x1], #16 sub w2, w2, #16 cbz w19, L_AES_XTS_decrypt_NEON_data_done and x16, x17, x9, asr 63 @@ -50366,157 +50366,157 @@ L_AES_XTS_decrypt_NEON_loop_nr_1 L_AES_XTS_decrypt_NEON_start_partial mov w2, w19 cbz w2, L_AES_XTS_decrypt_NEON_data_done - mov v2.d[0], x8 - mov v2.d[1], x9 + mov V2.D[0], x8 + mov V2.D[1], x9 and x16, x17, x9, asr 63 extr x11, x9, x8, #63 eor x10, x16, x8, lsl 1 - mov v1.d[0], x10 - mov v1.d[1], x11 + mov V1.D[0], x10 + mov V1.D[1], x11 mov x25, x4 - ld1 {v0.16b}, [x0], #16 - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v1.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v4.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V1.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_partial_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - sshr v10.16b, v0.16b, #7 - ushr v11.16b, v0.16b, #6 - ushr v8.16b, v0.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v0.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v0.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v0.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v0.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v0.16b - shl v0.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v0.4s, v9.4s, #24 - eor v0.16b, v0.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v0.16b, v0.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v0.16b, v0.16b, v9.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + sshr V10.16B, V0.16B, #7 + ushr V11.16B, V0.16B, #6 + ushr V8.16B, V0.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V0.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V0.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V0.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V0.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V0.16B + shl V0.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V0.4S, V9.4S, #24 + eor V0.16B, V0.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V0.16B, V0.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V0.16B, V0.16B, V9.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_partial_1 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v1.16b - st1 {v0.2d}, [x6] + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V1.16B + st1 {V0.2D}, [x6] add x1, x1, #16 mov w16, w2 L_AES_XTS_decrypt_NEON_start_byte @@ -50530,158 +50530,158 @@ L_AES_XTS_decrypt_NEON_start_byte sub x6, x6, x2 sub x1, x1, #16 mov x25, x4 - ld1 {v0.2d}, [x6] - ld1 {v4.2d}, [x25], #16 - eor v0.16b, v0.16b, v2.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v4.16b + ld1 {V0.2D}, [x6] + ld1 {V4.2D}, [x25], #16 + eor V0.16B, V0.16B, V2.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V4.16B sub w24, w7, #2 L_AES_XTS_decrypt_NEON_loop_nr_partial_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - sshr v10.16b, v0.16b, #7 - ushr v11.16b, v0.16b, #6 - ushr v8.16b, v0.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v0.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v0.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v0.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v0.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v0.16b - shl v0.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v0.4s, v9.4s, #24 - eor v0.16b, v0.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v0.16b, v0.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v0.16b, v0.16b, v9.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + sshr V10.16B, V0.16B, #7 + ushr V11.16B, V0.16B, #6 + ushr V8.16B, V0.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V0.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V0.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V0.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V0.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V0.16B + shl V0.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V0.4S, V9.4S, #24 + eor V0.16B, V0.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V0.16B, V0.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V0.16B, V0.16B, V9.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b + eor V0.16B, V0.16B, V4.16B subs w24, w24, #2 bne L_AES_XTS_decrypt_NEON_loop_nr_partial_2 - eor v8.16b, v0.16b, v12.16b - eor v9.16b, v0.16b, v13.16b - eor v10.16b, v0.16b, v14.16b - tbl v4.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v0.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v4.16b, v4.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v4.16b, v4.16b, v9.16b - tbl v4.16b, {v4.16b}, v3.16b - sshr v10.16b, v4.16b, #7 - ushr v11.16b, v4.16b, #6 - ushr v8.16b, v4.16b, #5 - and v10.16b, v10.16b, v15.16b - pmul v11.16b, v11.16b, v15.16b - pmul v8.16b, v8.16b, v15.16b - shl v9.16b, v4.16b, #1 - eor v10.16b, v10.16b, v9.16b - shl v9.16b, v4.16b, #3 - eor v8.16b, v8.16b, v9.16b - shl v9.16b, v4.16b, #2 - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v10.16b, v8.16b - eor v8.16b, v8.16b, v4.16b - eor v10.16b, v11.16b, v8.16b - eor v11.16b, v11.16b, v9.16b - eor v9.16b, v9.16b, v4.16b - shl v4.4s, v9.4s, #8 - rev32 v10.8h, v10.8h - sri v4.4s, v9.4s, #24 - eor v4.16b, v4.16b, v11.16b - shl v9.4s, v8.4s, #24 - eor v4.16b, v4.16b, v10.16b - sri v9.4s, v8.4s, #8 - eor v4.16b, v4.16b, v9.16b - ld1 {v0.2d}, [x25], #16 + eor V8.16B, V0.16B, V12.16B + eor V9.16B, V0.16B, V13.16B + eor V10.16B, V0.16B, V14.16B + tbl V4.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V0.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V4.16B, V4.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V4.16B, V4.16B, V9.16B + tbl V4.16B, {V4.16B}, V3.16B + sshr V10.16B, V4.16B, #7 + ushr V11.16B, V4.16B, #6 + ushr V8.16B, V4.16B, #5 + and V10.16B, V10.16B, V15.16B + pmul V11.16B, V11.16B, V15.16B + pmul V8.16B, V8.16B, V15.16B + shl V9.16B, V4.16B, #1 + eor V10.16B, V10.16B, V9.16B + shl V9.16B, V4.16B, #3 + eor V8.16B, V8.16B, V9.16B + shl V9.16B, V4.16B, #2 + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V10.16B, V8.16B + eor V8.16B, V8.16B, V4.16B + eor V10.16B, V11.16B, V8.16B + eor V11.16B, V11.16B, V9.16B + eor V9.16B, V9.16B, V4.16B + shl V4.4S, V9.4S, #8 + rev32 V10.8H, V10.8H + sri V4.4S, V9.4S, #24 + eor V4.16B, V4.16B, V11.16B + shl V9.4S, V8.4S, #24 + eor V4.16B, V4.16B, V10.16B + sri V9.4S, V8.4S, #8 + eor V4.16B, V4.16B, V9.16B + ld1 {V0.2D}, [x25], #16 ; XOR in Key Schedule - eor v4.16b, v4.16b, v0.16b - eor v8.16b, v4.16b, v12.16b - eor v9.16b, v4.16b, v13.16b - eor v10.16b, v4.16b, v14.16b - tbl v0.16b, {v16.16b, v17.16b, v18.16b, v19.16b}, v4.16b - tbl v8.16b, {v20.16b, v21.16b, v22.16b, v23.16b}, v8.16b - tbl v9.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v9.16b - tbl v10.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v10.16b - orr v0.16b, v0.16b, v8.16b - orr v9.16b, v9.16b, v10.16b - orr v0.16b, v0.16b, v9.16b - tbl v0.16b, {v0.16b}, v3.16b - ld1 {v4.2d}, [x25], #16 + eor V4.16B, V4.16B, V0.16B + eor V8.16B, V4.16B, V12.16B + eor V9.16B, V4.16B, V13.16B + eor V10.16B, V4.16B, V14.16B + tbl V0.16B, {V16.16B, V17.16B, V18.16B, V19.16B}, V4.16B + tbl V8.16B, {V20.16B, V21.16B, V22.16B, V23.16B}, V8.16B + tbl V9.16B, {V24.16B, V25.16B, V26.16B, V27.16B}, V9.16B + tbl V10.16B, {V28.16B, V29.16B, V30.16B, V31.16B}, V10.16B + orr V0.16B, V0.16B, V8.16B + orr V9.16B, V9.16B, V10.16B + orr V0.16B, V0.16B, V9.16B + tbl V0.16B, {V0.16B}, V3.16B + ld1 {V4.2D}, [x25], #16 ; XOR in Key Schedule - eor v0.16b, v0.16b, v4.16b - rev32 v0.16b, v0.16b - eor v0.16b, v0.16b, v2.16b - st1 {v0.16b}, [x1] + eor V0.16B, V0.16B, V4.16B + rev32 V0.16B, V0.16B + eor V0.16B, V0.16B, V2.16B + st1 {V0.16B}, [x1] L_AES_XTS_decrypt_NEON_data_done ldp x17, x19, [x29, #16] ldp x20, x21, [x29, #32] ldp x22, x23, [x29, #48] ldp x24, x25, [x29, #64] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] - ldp d12, d13, [x29, #112] - ldp d14, d15, [x29, #128] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] + ldp D12, D13, [x29, #112] + ldp D14, D15, [x29, #128] ldp x29, x30, [sp], #0x90 ret ENDP @@ -50690,7 +50690,7 @@ L_AES_XTS_decrypt_NEON_data_done ENDIF IF :LNOT::DEF:WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP IF :DEF:HAVE_AES_DECRYPT - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_td DCD 0x5051f4a7, 0x537e4165, 0xc31a17a4, 0x963a275e @@ -50759,7 +50759,7 @@ L_AES_ARM64_td DCD 0x617bcb84, 0x70d532b6, 0x74486c5c, 0x42d0b857 ENDIF IF :DEF:HAVE_AES_DECRYPT :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AESCCM :LOR: :DEF:HAVE_AESGCM :LOR: :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_te DCD 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b @@ -50941,7 +50941,7 @@ L_AES_invert_key_mix_loop ret ENDP ENDIF - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_rcon DCD 0x01000000, 0x02000000, 0x04000000, 0x08000000 @@ -52154,7 +52154,7 @@ L_AES_CTR_encrypt_loop_nr ENDIF IF :DEF:HAVE_AES_DECRYPT IF :DEF:WOLFSSL_AES_DIRECT :LOR: :DEF:WOLFSSL_AES_COUNTER :LOR: :DEF:HAVE_AES_CBC :LOR: :DEF:HAVE_AES_ECB - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_AES_ARM64_td4 DCB 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 @@ -53062,7 +53062,7 @@ L_AES_CBC_decrypt_end_dec ENDIF ENDIF IF :DEF:HAVE_AESGCM - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_GCM_gmult_len_r DCD 0x00000000, 0x1c200000, 0x38400000, 0x24600000 diff --git a/wolfcrypt/src/port/arm/armv8-chacha-asm.asm b/wolfcrypt/src/port/arm/armv8-chacha-asm.asm index 6a8a7e6565..842f25f3ff 100644 --- a/wolfcrypt/src/port/arm/armv8-chacha-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-chacha-asm.asm @@ -25,11 +25,11 @@ ; ruby ./chacha/chacha.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-chacha-asm.asm IF :DEF:HAVE_CHACHA - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_chacha20_arm64_ctr DCD 0x00000000, 0x00000001, 0x00000002, 0x00000003 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_chacha20_arm64_rol8 DCD 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f @@ -45,66 +45,66 @@ wc_chacha_crypt_bytes PROC stp x22, x23, [x29, #56] stp x24, x25, [x29, #72] str x26, [x29, #88] - stp d8, d9, [x29, #96] - stp d10, d11, [x29, #112] - stp d12, d13, [x29, #128] - stp d14, d15, [x29, #144] + stp D8, D9, [x29, #96] + stp D10, D11, [x29, #112] + stp D12, D13, [x29, #128] + stp D14, D15, [x29, #144] adrp x5, L_chacha20_arm64_rol8 add x5, x5, L_chacha20_arm64_rol8 adrp x6, L_chacha20_arm64_ctr add x6, x6, L_chacha20_arm64_ctr - eor v29.16b, v29.16b, v29.16b + eor V29.16B, V29.16B, V29.16B mov x26, #5 - eor v31.16b, v31.16b, v31.16b + eor V31.16B, V31.16B, V31.16B mov w7, #1 - ld1 {v30.16b}, [x5] - ld1 {v28.4s}, [x6] + ld1 {V30.16B}, [x5] + ld1 {V28.4S}, [x6] add x4, x0, #0x44 - mov v29.s[0], w26 - mov v31.s[0], w7 + mov V29.S[0], w26 + mov V31.S[0], w7 ; Load state to encrypt - ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x0] + ld1 {V16.4S, V17.4S, V18.4S, V19.4S}, [x0] cmp x3, #0x140 blt L_chacha_crypt_bytes_arm64_lt_320 mov w25, #4 L_chacha_crypt_bytes_arm64_loop_320 ; Move state into regular register - mov x8, v16.d[0] - mov x10, v16.d[1] - mov x12, v17.d[0] - mov x14, v17.d[1] - mov x16, v18.d[0] - mov x19, v18.d[1] - mov x21, v19.d[0] - mov x23, v19.d[1] + mov x8, V16.D[0] + mov x10, V16.D[1] + mov x12, V17.D[0] + mov x14, V17.D[1] + mov x16, V18.D[0] + mov x19, V18.D[1] + mov x21, V19.D[0] + mov x23, V19.D[1] sub x3, x3, #0x140 ; Move state into vector registers - dup v0.4s, v16.s[0] - dup v1.4s, v16.s[1] + dup V0.4S, V16.S[0] + dup V1.4S, V16.S[1] lsr x9, x8, #32 - dup v2.4s, v16.s[2] - dup v3.4s, v16.s[3] + dup V2.4S, V16.S[2] + dup V3.4S, V16.S[3] lsr x11, x10, #32 - dup v4.4s, v17.s[0] - dup v5.4s, v17.s[1] + dup V4.4S, V17.S[0] + dup V5.4S, V17.S[1] lsr x13, x12, #32 - dup v6.4s, v17.s[2] - dup v7.4s, v17.s[3] + dup V6.4S, V17.S[2] + dup V7.4S, V17.S[3] lsr x15, x14, #32 - dup v8.4s, v18.s[0] - dup v9.4s, v18.s[1] + dup V8.4S, V18.S[0] + dup V9.4S, V18.S[1] lsr x17, x16, #32 - dup v10.4s, v18.s[2] - dup v11.4s, v18.s[3] + dup V10.4S, V18.S[2] + dup V11.4S, V18.S[3] lsr x20, x19, #32 - dup v12.4s, v19.s[0] - dup v13.4s, v19.s[1] + dup V12.4S, V19.S[0] + dup V13.4S, V19.S[1] lsr x22, x21, #32 - dup v14.4s, v19.s[2] - dup v15.4s, v19.s[3] + dup V14.4S, V19.S[2] + dup V15.4S, V19.S[3] lsr x24, x23, #32 ; Add to counter word - add v12.4s, v12.4s, v28.4s + add V12.4S, V12.4S, V28.4S add w21, w21, w25 ; Set number of odd+even rounds to perform mov x26, #10 @@ -112,684 +112,684 @@ L_chacha_crypt_bytes_arm64_round_start_320 subs x26, x26, #1 ; Round odd ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v4.4s + add V0.4S, V0.4S, V4.4S add w8, w8, w12 - add v1.4s, v1.4s, v5.4s + add V1.4S, V1.4S, V5.4S add w9, w9, w13 - add v2.4s, v2.4s, v6.4s + add V2.4S, V2.4S, V6.4S add w10, w10, w14 - add v3.4s, v3.4s, v7.4s + add V3.4S, V3.4S, V7.4S add w11, w11, w15 - eor v12.16b, v12.16b, v0.16b + eor V12.16B, V12.16B, V0.16B eor w21, w21, w8 - eor v13.16b, v13.16b, v1.16b + eor V13.16B, V13.16B, V1.16B eor w22, w22, w9 - eor v14.16b, v14.16b, v2.16b + eor V14.16B, V14.16B, V2.16B eor w23, w23, w10 - eor v15.16b, v15.16b, v3.16b + eor V15.16B, V15.16B, V3.16B eor w24, w24, w11 - rev32 v12.8h, v12.8h + rev32 V12.8H, V12.8H ror w21, w21, #16 - rev32 v13.8h, v13.8h + rev32 V13.8H, V13.8H ror w22, w22, #16 - rev32 v14.8h, v14.8h + rev32 V14.8H, V14.8H ror w23, w23, #16 - rev32 v15.8h, v15.8h + rev32 V15.8H, V15.8H ror w24, w24, #16 ; c += d; b ^= c; b <<<= 12; - add v8.4s, v8.4s, v12.4s + add V8.4S, V8.4S, V12.4S add w16, w16, w21 - add v9.4s, v9.4s, v13.4s + add V9.4S, V9.4S, V13.4S add w17, w17, w22 - add v10.4s, v10.4s, v14.4s + add V10.4S, V10.4S, V14.4S add w19, w19, w23 - add v11.4s, v11.4s, v15.4s + add V11.4S, V11.4S, V15.4S add w20, w20, w24 - eor v20.16b, v4.16b, v8.16b + eor V20.16B, V4.16B, V8.16B eor w12, w12, w16 - eor v21.16b, v5.16b, v9.16b + eor V21.16B, V5.16B, V9.16B eor w13, w13, w17 - eor v22.16b, v6.16b, v10.16b + eor V22.16B, V6.16B, V10.16B eor w14, w14, w19 - eor v23.16b, v7.16b, v11.16b + eor V23.16B, V7.16B, V11.16B eor w15, w15, w20 - shl v4.4s, v20.4s, #12 + shl V4.4S, V20.4S, #12 ror w12, w12, #20 - shl v5.4s, v21.4s, #12 + shl V5.4S, V21.4S, #12 ror w13, w13, #20 - shl v6.4s, v22.4s, #12 + shl V6.4S, V22.4S, #12 ror w14, w14, #20 - shl v7.4s, v23.4s, #12 + shl V7.4S, V23.4S, #12 ror w15, w15, #20 - sri v4.4s, v20.4s, #20 - sri v5.4s, v21.4s, #20 - sri v6.4s, v22.4s, #20 - sri v7.4s, v23.4s, #20 + sri V4.4S, V20.4S, #20 + sri V5.4S, V21.4S, #20 + sri V6.4S, V22.4S, #20 + sri V7.4S, V23.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v4.4s + add V0.4S, V0.4S, V4.4S add w8, w8, w12 - add v1.4s, v1.4s, v5.4s + add V1.4S, V1.4S, V5.4S add w9, w9, w13 - add v2.4s, v2.4s, v6.4s + add V2.4S, V2.4S, V6.4S add w10, w10, w14 - add v3.4s, v3.4s, v7.4s + add V3.4S, V3.4S, V7.4S add w11, w11, w15 - eor v12.16b, v12.16b, v0.16b + eor V12.16B, V12.16B, V0.16B eor w21, w21, w8 - eor v13.16b, v13.16b, v1.16b + eor V13.16B, V13.16B, V1.16B eor w22, w22, w9 - eor v14.16b, v14.16b, v2.16b + eor V14.16B, V14.16B, V2.16B eor w23, w23, w10 - eor v15.16b, v15.16b, v3.16b + eor V15.16B, V15.16B, V3.16B eor w24, w24, w11 - tbl v12.16b, {v12.16b}, v30.16b + tbl V12.16B, {V12.16B}, V30.16B ror w21, w21, #24 - tbl v13.16b, {v13.16b}, v30.16b + tbl V13.16B, {V13.16B}, V30.16B ror w22, w22, #24 - tbl v14.16b, {v14.16b}, v30.16b + tbl V14.16B, {V14.16B}, V30.16B ror w23, w23, #24 - tbl v15.16b, {v15.16b}, v30.16b + tbl V15.16B, {V15.16B}, V30.16B ror w24, w24, #24 ; c += d; b ^= c; b <<<= 7; - add v8.4s, v8.4s, v12.4s + add V8.4S, V8.4S, V12.4S add w16, w16, w21 - add v9.4s, v9.4s, v13.4s + add V9.4S, V9.4S, V13.4S add w17, w17, w22 - add v10.4s, v10.4s, v14.4s + add V10.4S, V10.4S, V14.4S add w19, w19, w23 - add v11.4s, v11.4s, v15.4s + add V11.4S, V11.4S, V15.4S add w20, w20, w24 - eor v20.16b, v4.16b, v8.16b + eor V20.16B, V4.16B, V8.16B eor w12, w12, w16 - eor v21.16b, v5.16b, v9.16b + eor V21.16B, V5.16B, V9.16B eor w13, w13, w17 - eor v22.16b, v6.16b, v10.16b + eor V22.16B, V6.16B, V10.16B eor w14, w14, w19 - eor v23.16b, v7.16b, v11.16b + eor V23.16B, V7.16B, V11.16B eor w15, w15, w20 - shl v4.4s, v20.4s, #7 + shl V4.4S, V20.4S, #7 ror w12, w12, #25 - shl v5.4s, v21.4s, #7 + shl V5.4S, V21.4S, #7 ror w13, w13, #25 - shl v6.4s, v22.4s, #7 + shl V6.4S, V22.4S, #7 ror w14, w14, #25 - shl v7.4s, v23.4s, #7 + shl V7.4S, V23.4S, #7 ror w15, w15, #25 - sri v4.4s, v20.4s, #25 - sri v5.4s, v21.4s, #25 - sri v6.4s, v22.4s, #25 - sri v7.4s, v23.4s, #25 + sri V4.4S, V20.4S, #25 + sri V5.4S, V21.4S, #25 + sri V6.4S, V22.4S, #25 + sri V7.4S, V23.4S, #25 ; Round even ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v5.4s + add V0.4S, V0.4S, V5.4S add w8, w8, w13 - add v1.4s, v1.4s, v6.4s + add V1.4S, V1.4S, V6.4S add w9, w9, w14 - add v2.4s, v2.4s, v7.4s + add V2.4S, V2.4S, V7.4S add w10, w10, w15 - add v3.4s, v3.4s, v4.4s + add V3.4S, V3.4S, V4.4S add w11, w11, w12 - eor v15.16b, v15.16b, v0.16b + eor V15.16B, V15.16B, V0.16B eor w24, w24, w8 - eor v12.16b, v12.16b, v1.16b + eor V12.16B, V12.16B, V1.16B eor w21, w21, w9 - eor v13.16b, v13.16b, v2.16b + eor V13.16B, V13.16B, V2.16B eor w22, w22, w10 - eor v14.16b, v14.16b, v3.16b + eor V14.16B, V14.16B, V3.16B eor w23, w23, w11 - rev32 v15.8h, v15.8h + rev32 V15.8H, V15.8H ror w24, w24, #16 - rev32 v12.8h, v12.8h + rev32 V12.8H, V12.8H ror w21, w21, #16 - rev32 v13.8h, v13.8h + rev32 V13.8H, V13.8H ror w22, w22, #16 - rev32 v14.8h, v14.8h + rev32 V14.8H, V14.8H ror w23, w23, #16 ; c += d; b ^= c; b <<<= 12; - add v10.4s, v10.4s, v15.4s + add V10.4S, V10.4S, V15.4S add w19, w19, w24 - add v11.4s, v11.4s, v12.4s + add V11.4S, V11.4S, V12.4S add w20, w20, w21 - add v8.4s, v8.4s, v13.4s + add V8.4S, V8.4S, V13.4S add w16, w16, w22 - add v9.4s, v9.4s, v14.4s + add V9.4S, V9.4S, V14.4S add w17, w17, w23 - eor v20.16b, v5.16b, v10.16b + eor V20.16B, V5.16B, V10.16B eor w13, w13, w19 - eor v21.16b, v6.16b, v11.16b + eor V21.16B, V6.16B, V11.16B eor w14, w14, w20 - eor v22.16b, v7.16b, v8.16b + eor V22.16B, V7.16B, V8.16B eor w15, w15, w16 - eor v23.16b, v4.16b, v9.16b + eor V23.16B, V4.16B, V9.16B eor w12, w12, w17 - shl v5.4s, v20.4s, #12 + shl V5.4S, V20.4S, #12 ror w13, w13, #20 - shl v6.4s, v21.4s, #12 + shl V6.4S, V21.4S, #12 ror w14, w14, #20 - shl v7.4s, v22.4s, #12 + shl V7.4S, V22.4S, #12 ror w15, w15, #20 - shl v4.4s, v23.4s, #12 + shl V4.4S, V23.4S, #12 ror w12, w12, #20 - sri v5.4s, v20.4s, #20 - sri v6.4s, v21.4s, #20 - sri v7.4s, v22.4s, #20 - sri v4.4s, v23.4s, #20 + sri V5.4S, V20.4S, #20 + sri V6.4S, V21.4S, #20 + sri V7.4S, V22.4S, #20 + sri V4.4S, V23.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v5.4s + add V0.4S, V0.4S, V5.4S add w8, w8, w13 - add v1.4s, v1.4s, v6.4s + add V1.4S, V1.4S, V6.4S add w9, w9, w14 - add v2.4s, v2.4s, v7.4s + add V2.4S, V2.4S, V7.4S add w10, w10, w15 - add v3.4s, v3.4s, v4.4s + add V3.4S, V3.4S, V4.4S add w11, w11, w12 - eor v15.16b, v15.16b, v0.16b + eor V15.16B, V15.16B, V0.16B eor w24, w24, w8 - eor v12.16b, v12.16b, v1.16b + eor V12.16B, V12.16B, V1.16B eor w21, w21, w9 - eor v13.16b, v13.16b, v2.16b + eor V13.16B, V13.16B, V2.16B eor w22, w22, w10 - eor v14.16b, v14.16b, v3.16b + eor V14.16B, V14.16B, V3.16B eor w23, w23, w11 - tbl v15.16b, {v15.16b}, v30.16b + tbl V15.16B, {V15.16B}, V30.16B ror w24, w24, #24 - tbl v12.16b, {v12.16b}, v30.16b + tbl V12.16B, {V12.16B}, V30.16B ror w21, w21, #24 - tbl v13.16b, {v13.16b}, v30.16b + tbl V13.16B, {V13.16B}, V30.16B ror w22, w22, #24 - tbl v14.16b, {v14.16b}, v30.16b + tbl V14.16B, {V14.16B}, V30.16B ror w23, w23, #24 ; c += d; b ^= c; b <<<= 7; - add v10.4s, v10.4s, v15.4s + add V10.4S, V10.4S, V15.4S add w19, w19, w24 - add v11.4s, v11.4s, v12.4s + add V11.4S, V11.4S, V12.4S add w20, w20, w21 - add v8.4s, v8.4s, v13.4s + add V8.4S, V8.4S, V13.4S add w16, w16, w22 - add v9.4s, v9.4s, v14.4s + add V9.4S, V9.4S, V14.4S add w17, w17, w23 - eor v20.16b, v5.16b, v10.16b + eor V20.16B, V5.16B, V10.16B eor w13, w13, w19 - eor v21.16b, v6.16b, v11.16b + eor V21.16B, V6.16B, V11.16B eor w14, w14, w20 - eor v22.16b, v7.16b, v8.16b + eor V22.16B, V7.16B, V8.16B eor w15, w15, w16 - eor v23.16b, v4.16b, v9.16b + eor V23.16B, V4.16B, V9.16B eor w12, w12, w17 - shl v5.4s, v20.4s, #7 + shl V5.4S, V20.4S, #7 ror w13, w13, #25 - shl v6.4s, v21.4s, #7 + shl V6.4S, V21.4S, #7 ror w14, w14, #25 - shl v7.4s, v22.4s, #7 + shl V7.4S, V22.4S, #7 ror w15, w15, #25 - shl v4.4s, v23.4s, #7 + shl V4.4S, V23.4S, #7 ror w12, w12, #25 - sri v5.4s, v20.4s, #25 - sri v6.4s, v21.4s, #25 - sri v7.4s, v22.4s, #25 - sri v4.4s, v23.4s, #25 + sri V5.4S, V20.4S, #25 + sri V6.4S, V21.4S, #25 + sri V7.4S, V22.4S, #25 + sri V4.4S, V23.4S, #25 bne L_chacha_crypt_bytes_arm64_round_start_320 ; Add counter now rather than after transposed - add v12.4s, v12.4s, v28.4s + add V12.4S, V12.4S, V28.4S add w21, w21, w25 ; Load message - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 ; Transpose vectors - trn1 v20.4s, v0.4s, v1.4s - trn1 v22.4s, v2.4s, v3.4s + trn1 V20.4S, V0.4S, V1.4S + trn1 V22.4S, V2.4S, V3.4S orr x8, x8, x9, lsl 32 - trn2 v21.4s, v0.4s, v1.4s - trn2 v23.4s, v2.4s, v3.4s - trn1 v0.2d, v20.2d, v22.2d - trn1 v1.2d, v21.2d, v23.2d + trn2 V21.4S, V0.4S, V1.4S + trn2 V23.4S, V2.4S, V3.4S + trn1 V0.2D, V20.2D, V22.2D + trn1 V1.2D, V21.2D, V23.2D orr x10, x10, x11, lsl 32 - trn2 v2.2d, v20.2d, v22.2d - trn2 v3.2d, v21.2d, v23.2d - trn1 v20.4s, v4.4s, v5.4s - trn1 v22.4s, v6.4s, v7.4s + trn2 V2.2D, V20.2D, V22.2D + trn2 V3.2D, V21.2D, V23.2D + trn1 V20.4S, V4.4S, V5.4S + trn1 V22.4S, V6.4S, V7.4S orr x12, x12, x13, lsl 32 - trn2 v21.4s, v4.4s, v5.4s - trn2 v23.4s, v6.4s, v7.4s - trn1 v4.2d, v20.2d, v22.2d - trn1 v5.2d, v21.2d, v23.2d + trn2 V21.4S, V4.4S, V5.4S + trn2 V23.4S, V6.4S, V7.4S + trn1 V4.2D, V20.2D, V22.2D + trn1 V5.2D, V21.2D, V23.2D orr x14, x14, x15, lsl 32 - trn2 v6.2d, v20.2d, v22.2d - trn2 v7.2d, v21.2d, v23.2d - trn1 v20.4s, v8.4s, v9.4s - trn1 v22.4s, v10.4s, v11.4s + trn2 V6.2D, V20.2D, V22.2D + trn2 V7.2D, V21.2D, V23.2D + trn1 V20.4S, V8.4S, V9.4S + trn1 V22.4S, V10.4S, V11.4S orr x16, x16, x17, lsl 32 - trn2 v21.4s, v8.4s, v9.4s - trn2 v23.4s, v10.4s, v11.4s - trn1 v8.2d, v20.2d, v22.2d - trn1 v9.2d, v21.2d, v23.2d + trn2 V21.4S, V8.4S, V9.4S + trn2 V23.4S, V10.4S, V11.4S + trn1 V8.2D, V20.2D, V22.2D + trn1 V9.2D, V21.2D, V23.2D orr x19, x19, x20, lsl 32 - trn2 v10.2d, v20.2d, v22.2d - trn2 v11.2d, v21.2d, v23.2d - trn1 v20.4s, v12.4s, v13.4s - trn1 v22.4s, v14.4s, v15.4s + trn2 V10.2D, V20.2D, V22.2D + trn2 V11.2D, V21.2D, V23.2D + trn1 V20.4S, V12.4S, V13.4S + trn1 V22.4S, V14.4S, V15.4S orr x21, x21, x22, lsl 32 - trn2 v21.4s, v12.4s, v13.4s - trn2 v23.4s, v14.4s, v15.4s - trn1 v12.2d, v20.2d, v22.2d - trn1 v13.2d, v21.2d, v23.2d + trn2 V21.4S, V12.4S, V13.4S + trn2 V23.4S, V14.4S, V15.4S + trn1 V12.2D, V20.2D, V22.2D + trn1 V13.2D, V21.2D, V23.2D orr x23, x23, x24, lsl 32 - trn2 v14.2d, v20.2d, v22.2d - trn2 v15.2d, v21.2d, v23.2d + trn2 V14.2D, V20.2D, V22.2D + trn2 V15.2D, V21.2D, V23.2D ; Add back state, XOR in message and store (load next block) - add v20.4s, v0.4s, v16.4s - add v21.4s, v4.4s, v17.4s - add v22.4s, v8.4s, v18.4s - add v23.4s, v12.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v1.4s, v16.4s - add v21.4s, v5.4s, v17.4s - add v22.4s, v9.4s, v18.4s - add v23.4s, v13.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v2.4s, v16.4s - add v21.4s, v6.4s, v17.4s - add v22.4s, v10.4s, v18.4s - add v23.4s, v14.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v3.4s, v16.4s - add v21.4s, v7.4s, v17.4s - add v22.4s, v11.4s, v18.4s - add v23.4s, v15.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 + add V20.4S, V0.4S, V16.4S + add V21.4S, V4.4S, V17.4S + add V22.4S, V8.4S, V18.4S + add V23.4S, V12.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V1.4S, V16.4S + add V21.4S, V5.4S, V17.4S + add V22.4S, V9.4S, V18.4S + add V23.4S, V13.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V2.4S, V16.4S + add V21.4S, V6.4S, V17.4S + add V22.4S, V10.4S, V18.4S + add V23.4S, V14.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V3.4S, V16.4S + add V21.4S, V7.4S, V17.4S + add V22.4S, V11.4S, V18.4S + add V23.4S, V15.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 ; Move regular registers into vector registers for adding and xor - mov v0.d[0], x8 - mov v0.d[1], x10 - mov v1.d[0], x12 - mov v1.d[1], x14 - mov v2.d[0], x16 - mov v2.d[1], x19 - mov v3.d[0], x21 - mov v3.d[1], x23 + mov V0.D[0], x8 + mov V0.D[1], x10 + mov V1.D[0], x12 + mov V1.D[1], x14 + mov V2.D[0], x16 + mov V2.D[1], x19 + mov V3.D[0], x21 + mov V3.D[1], x23 ; Add back state, XOR in message and store - add v0.4s, v0.4s, v16.4s - add v1.4s, v1.4s, v17.4s - add v2.4s, v2.4s, v18.4s - add v3.4s, v3.4s, v19.4s - eor v0.16b, v0.16b, v24.16b - eor v1.16b, v1.16b, v25.16b - eor v2.16b, v2.16b, v26.16b - eor v3.16b, v3.16b, v27.16b - st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x1], #0x40 + add V0.4S, V0.4S, V16.4S + add V1.4S, V1.4S, V17.4S + add V2.4S, V2.4S, V18.4S + add V3.4S, V3.4S, V19.4S + eor V0.16B, V0.16B, V24.16B + eor V1.16B, V1.16B, V25.16B + eor V2.16B, V2.16B, V26.16B + eor V3.16B, V3.16B, V27.16B + st1 {V0.4S, V1.4S, V2.4S, V3.4S}, [x1], #0x40 cmp x3, #0x140 - add v19.4s, v19.4s, v29.4s + add V19.4S, V19.4S, V29.4S bge L_chacha_crypt_bytes_arm64_loop_320 ; Done doing 320 bytes at a time L_chacha_crypt_bytes_arm64_lt_320 cmp x3, #0x100 blt L_chacha_crypt_bytes_arm64_lt_256 ; Move state into vector registers - dup v0.4s, v16.s[0] - dup v1.4s, v16.s[1] - dup v2.4s, v16.s[2] - dup v3.4s, v16.s[3] - dup v4.4s, v17.s[0] - dup v5.4s, v17.s[1] - dup v6.4s, v17.s[2] - dup v7.4s, v17.s[3] - dup v8.4s, v18.s[0] - dup v9.4s, v18.s[1] - dup v10.4s, v18.s[2] - dup v11.4s, v18.s[3] - dup v12.4s, v19.s[0] - dup v13.4s, v19.s[1] - dup v14.4s, v19.s[2] - dup v15.4s, v19.s[3] + dup V0.4S, V16.S[0] + dup V1.4S, V16.S[1] + dup V2.4S, V16.S[2] + dup V3.4S, V16.S[3] + dup V4.4S, V17.S[0] + dup V5.4S, V17.S[1] + dup V6.4S, V17.S[2] + dup V7.4S, V17.S[3] + dup V8.4S, V18.S[0] + dup V9.4S, V18.S[1] + dup V10.4S, V18.S[2] + dup V11.4S, V18.S[3] + dup V12.4S, V19.S[0] + dup V13.4S, V19.S[1] + dup V14.4S, V19.S[2] + dup V15.4S, V19.S[3] ; Add to counter word - add v12.4s, v12.4s, v28.4s + add V12.4S, V12.4S, V28.4S ; Set number of odd+even rounds to perform mov x26, #10 L_chacha_crypt_bytes_arm64_round_start_256 subs x26, x26, #1 ; Round odd ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v4.4s - add v1.4s, v1.4s, v5.4s - add v2.4s, v2.4s, v6.4s - add v3.4s, v3.4s, v7.4s - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - rev32 v12.8h, v12.8h - rev32 v13.8h, v13.8h - rev32 v14.8h, v14.8h - rev32 v15.8h, v15.8h + add V0.4S, V0.4S, V4.4S + add V1.4S, V1.4S, V5.4S + add V2.4S, V2.4S, V6.4S + add V3.4S, V3.4S, V7.4S + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + rev32 V12.8H, V12.8H + rev32 V13.8H, V13.8H + rev32 V14.8H, V14.8H + rev32 V15.8H, V15.8H ; c += d; b ^= c; b <<<= 12; - add v8.4s, v8.4s, v12.4s - add v9.4s, v9.4s, v13.4s - add v10.4s, v10.4s, v14.4s - add v11.4s, v11.4s, v15.4s - eor v20.16b, v4.16b, v8.16b - eor v21.16b, v5.16b, v9.16b - eor v22.16b, v6.16b, v10.16b - eor v23.16b, v7.16b, v11.16b - shl v4.4s, v20.4s, #12 - shl v5.4s, v21.4s, #12 - shl v6.4s, v22.4s, #12 - shl v7.4s, v23.4s, #12 - sri v4.4s, v20.4s, #20 - sri v5.4s, v21.4s, #20 - sri v6.4s, v22.4s, #20 - sri v7.4s, v23.4s, #20 + add V8.4S, V8.4S, V12.4S + add V9.4S, V9.4S, V13.4S + add V10.4S, V10.4S, V14.4S + add V11.4S, V11.4S, V15.4S + eor V20.16B, V4.16B, V8.16B + eor V21.16B, V5.16B, V9.16B + eor V22.16B, V6.16B, V10.16B + eor V23.16B, V7.16B, V11.16B + shl V4.4S, V20.4S, #12 + shl V5.4S, V21.4S, #12 + shl V6.4S, V22.4S, #12 + shl V7.4S, V23.4S, #12 + sri V4.4S, V20.4S, #20 + sri V5.4S, V21.4S, #20 + sri V6.4S, V22.4S, #20 + sri V7.4S, V23.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v4.4s - add v1.4s, v1.4s, v5.4s - add v2.4s, v2.4s, v6.4s - add v3.4s, v3.4s, v7.4s - eor v12.16b, v12.16b, v0.16b - eor v13.16b, v13.16b, v1.16b - eor v14.16b, v14.16b, v2.16b - eor v15.16b, v15.16b, v3.16b - tbl v12.16b, {v12.16b}, v30.16b - tbl v13.16b, {v13.16b}, v30.16b - tbl v14.16b, {v14.16b}, v30.16b - tbl v15.16b, {v15.16b}, v30.16b + add V0.4S, V0.4S, V4.4S + add V1.4S, V1.4S, V5.4S + add V2.4S, V2.4S, V6.4S + add V3.4S, V3.4S, V7.4S + eor V12.16B, V12.16B, V0.16B + eor V13.16B, V13.16B, V1.16B + eor V14.16B, V14.16B, V2.16B + eor V15.16B, V15.16B, V3.16B + tbl V12.16B, {V12.16B}, V30.16B + tbl V13.16B, {V13.16B}, V30.16B + tbl V14.16B, {V14.16B}, V30.16B + tbl V15.16B, {V15.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v8.4s, v8.4s, v12.4s - add v9.4s, v9.4s, v13.4s - add v10.4s, v10.4s, v14.4s - add v11.4s, v11.4s, v15.4s - eor v20.16b, v4.16b, v8.16b - eor v21.16b, v5.16b, v9.16b - eor v22.16b, v6.16b, v10.16b - eor v23.16b, v7.16b, v11.16b - shl v4.4s, v20.4s, #7 - shl v5.4s, v21.4s, #7 - shl v6.4s, v22.4s, #7 - shl v7.4s, v23.4s, #7 - sri v4.4s, v20.4s, #25 - sri v5.4s, v21.4s, #25 - sri v6.4s, v22.4s, #25 - sri v7.4s, v23.4s, #25 + add V8.4S, V8.4S, V12.4S + add V9.4S, V9.4S, V13.4S + add V10.4S, V10.4S, V14.4S + add V11.4S, V11.4S, V15.4S + eor V20.16B, V4.16B, V8.16B + eor V21.16B, V5.16B, V9.16B + eor V22.16B, V6.16B, V10.16B + eor V23.16B, V7.16B, V11.16B + shl V4.4S, V20.4S, #7 + shl V5.4S, V21.4S, #7 + shl V6.4S, V22.4S, #7 + shl V7.4S, V23.4S, #7 + sri V4.4S, V20.4S, #25 + sri V5.4S, V21.4S, #25 + sri V6.4S, V22.4S, #25 + sri V7.4S, V23.4S, #25 ; Round even ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v5.4s - add v1.4s, v1.4s, v6.4s - add v2.4s, v2.4s, v7.4s - add v3.4s, v3.4s, v4.4s - eor v15.16b, v15.16b, v0.16b - eor v12.16b, v12.16b, v1.16b - eor v13.16b, v13.16b, v2.16b - eor v14.16b, v14.16b, v3.16b - rev32 v15.8h, v15.8h - rev32 v12.8h, v12.8h - rev32 v13.8h, v13.8h - rev32 v14.8h, v14.8h + add V0.4S, V0.4S, V5.4S + add V1.4S, V1.4S, V6.4S + add V2.4S, V2.4S, V7.4S + add V3.4S, V3.4S, V4.4S + eor V15.16B, V15.16B, V0.16B + eor V12.16B, V12.16B, V1.16B + eor V13.16B, V13.16B, V2.16B + eor V14.16B, V14.16B, V3.16B + rev32 V15.8H, V15.8H + rev32 V12.8H, V12.8H + rev32 V13.8H, V13.8H + rev32 V14.8H, V14.8H ; c += d; b ^= c; b <<<= 12; - add v10.4s, v10.4s, v15.4s - add v11.4s, v11.4s, v12.4s - add v8.4s, v8.4s, v13.4s - add v9.4s, v9.4s, v14.4s - eor v20.16b, v5.16b, v10.16b - eor v21.16b, v6.16b, v11.16b - eor v22.16b, v7.16b, v8.16b - eor v23.16b, v4.16b, v9.16b - shl v5.4s, v20.4s, #12 - shl v6.4s, v21.4s, #12 - shl v7.4s, v22.4s, #12 - shl v4.4s, v23.4s, #12 - sri v5.4s, v20.4s, #20 - sri v6.4s, v21.4s, #20 - sri v7.4s, v22.4s, #20 - sri v4.4s, v23.4s, #20 + add V10.4S, V10.4S, V15.4S + add V11.4S, V11.4S, V12.4S + add V8.4S, V8.4S, V13.4S + add V9.4S, V9.4S, V14.4S + eor V20.16B, V5.16B, V10.16B + eor V21.16B, V6.16B, V11.16B + eor V22.16B, V7.16B, V8.16B + eor V23.16B, V4.16B, V9.16B + shl V5.4S, V20.4S, #12 + shl V6.4S, V21.4S, #12 + shl V7.4S, V22.4S, #12 + shl V4.4S, V23.4S, #12 + sri V5.4S, V20.4S, #20 + sri V6.4S, V21.4S, #20 + sri V7.4S, V22.4S, #20 + sri V4.4S, V23.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v5.4s - add v1.4s, v1.4s, v6.4s - add v2.4s, v2.4s, v7.4s - add v3.4s, v3.4s, v4.4s - eor v15.16b, v15.16b, v0.16b - eor v12.16b, v12.16b, v1.16b - eor v13.16b, v13.16b, v2.16b - eor v14.16b, v14.16b, v3.16b - tbl v15.16b, {v15.16b}, v30.16b - tbl v12.16b, {v12.16b}, v30.16b - tbl v13.16b, {v13.16b}, v30.16b - tbl v14.16b, {v14.16b}, v30.16b + add V0.4S, V0.4S, V5.4S + add V1.4S, V1.4S, V6.4S + add V2.4S, V2.4S, V7.4S + add V3.4S, V3.4S, V4.4S + eor V15.16B, V15.16B, V0.16B + eor V12.16B, V12.16B, V1.16B + eor V13.16B, V13.16B, V2.16B + eor V14.16B, V14.16B, V3.16B + tbl V15.16B, {V15.16B}, V30.16B + tbl V12.16B, {V12.16B}, V30.16B + tbl V13.16B, {V13.16B}, V30.16B + tbl V14.16B, {V14.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v10.4s, v10.4s, v15.4s - add v11.4s, v11.4s, v12.4s - add v8.4s, v8.4s, v13.4s - add v9.4s, v9.4s, v14.4s - eor v20.16b, v5.16b, v10.16b - eor v21.16b, v6.16b, v11.16b - eor v22.16b, v7.16b, v8.16b - eor v23.16b, v4.16b, v9.16b - shl v5.4s, v20.4s, #7 - shl v6.4s, v21.4s, #7 - shl v7.4s, v22.4s, #7 - shl v4.4s, v23.4s, #7 - sri v5.4s, v20.4s, #25 - sri v6.4s, v21.4s, #25 - sri v7.4s, v22.4s, #25 - sri v4.4s, v23.4s, #25 + add V10.4S, V10.4S, V15.4S + add V11.4S, V11.4S, V12.4S + add V8.4S, V8.4S, V13.4S + add V9.4S, V9.4S, V14.4S + eor V20.16B, V5.16B, V10.16B + eor V21.16B, V6.16B, V11.16B + eor V22.16B, V7.16B, V8.16B + eor V23.16B, V4.16B, V9.16B + shl V5.4S, V20.4S, #7 + shl V6.4S, V21.4S, #7 + shl V7.4S, V22.4S, #7 + shl V4.4S, V23.4S, #7 + sri V5.4S, V20.4S, #25 + sri V6.4S, V21.4S, #25 + sri V7.4S, V22.4S, #25 + sri V4.4S, V23.4S, #25 bne L_chacha_crypt_bytes_arm64_round_start_256 mov x26, #4 ; Add counter now rather than after transposed - add v12.4s, v12.4s, v28.4s + add V12.4S, V12.4S, V28.4S ; Load message - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 ; Transpose vectors - trn1 v20.4s, v0.4s, v1.4s - trn1 v22.4s, v2.4s, v3.4s - trn2 v21.4s, v0.4s, v1.4s - trn2 v23.4s, v2.4s, v3.4s - trn1 v0.2d, v20.2d, v22.2d - trn1 v1.2d, v21.2d, v23.2d - trn2 v2.2d, v20.2d, v22.2d - trn2 v3.2d, v21.2d, v23.2d - trn1 v20.4s, v4.4s, v5.4s - trn1 v22.4s, v6.4s, v7.4s - trn2 v21.4s, v4.4s, v5.4s - trn2 v23.4s, v6.4s, v7.4s - trn1 v4.2d, v20.2d, v22.2d - trn1 v5.2d, v21.2d, v23.2d - trn2 v6.2d, v20.2d, v22.2d - trn2 v7.2d, v21.2d, v23.2d - trn1 v20.4s, v8.4s, v9.4s - trn1 v22.4s, v10.4s, v11.4s - trn2 v21.4s, v8.4s, v9.4s - trn2 v23.4s, v10.4s, v11.4s - trn1 v8.2d, v20.2d, v22.2d - trn1 v9.2d, v21.2d, v23.2d - trn2 v10.2d, v20.2d, v22.2d - trn2 v11.2d, v21.2d, v23.2d - trn1 v20.4s, v12.4s, v13.4s - trn1 v22.4s, v14.4s, v15.4s - trn2 v21.4s, v12.4s, v13.4s - trn2 v23.4s, v14.4s, v15.4s - trn1 v12.2d, v20.2d, v22.2d - trn1 v13.2d, v21.2d, v23.2d - trn2 v14.2d, v20.2d, v22.2d - trn2 v15.2d, v21.2d, v23.2d + trn1 V20.4S, V0.4S, V1.4S + trn1 V22.4S, V2.4S, V3.4S + trn2 V21.4S, V0.4S, V1.4S + trn2 V23.4S, V2.4S, V3.4S + trn1 V0.2D, V20.2D, V22.2D + trn1 V1.2D, V21.2D, V23.2D + trn2 V2.2D, V20.2D, V22.2D + trn2 V3.2D, V21.2D, V23.2D + trn1 V20.4S, V4.4S, V5.4S + trn1 V22.4S, V6.4S, V7.4S + trn2 V21.4S, V4.4S, V5.4S + trn2 V23.4S, V6.4S, V7.4S + trn1 V4.2D, V20.2D, V22.2D + trn1 V5.2D, V21.2D, V23.2D + trn2 V6.2D, V20.2D, V22.2D + trn2 V7.2D, V21.2D, V23.2D + trn1 V20.4S, V8.4S, V9.4S + trn1 V22.4S, V10.4S, V11.4S + trn2 V21.4S, V8.4S, V9.4S + trn2 V23.4S, V10.4S, V11.4S + trn1 V8.2D, V20.2D, V22.2D + trn1 V9.2D, V21.2D, V23.2D + trn2 V10.2D, V20.2D, V22.2D + trn2 V11.2D, V21.2D, V23.2D + trn1 V20.4S, V12.4S, V13.4S + trn1 V22.4S, V14.4S, V15.4S + trn2 V21.4S, V12.4S, V13.4S + trn2 V23.4S, V14.4S, V15.4S + trn1 V12.2D, V20.2D, V22.2D + trn1 V13.2D, V21.2D, V23.2D + trn2 V14.2D, V20.2D, V22.2D + trn2 V15.2D, V21.2D, V23.2D ; Add back state, XOR in message and store (load next block) - add v20.4s, v0.4s, v16.4s - add v21.4s, v4.4s, v17.4s - add v22.4s, v8.4s, v18.4s - add v23.4s, v12.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v1.4s, v16.4s - add v21.4s, v5.4s, v17.4s - add v22.4s, v9.4s, v18.4s - add v23.4s, v13.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v2.4s, v16.4s - add v21.4s, v6.4s, v17.4s - add v22.4s, v10.4s, v18.4s - add v23.4s, v14.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v20.4s, v3.4s, v16.4s - add v21.4s, v7.4s, v17.4s - add v22.4s, v11.4s, v18.4s - add v23.4s, v15.4s, v19.4s - eor v20.16b, v20.16b, v24.16b - eor v21.16b, v21.16b, v25.16b - eor v22.16b, v22.16b, v26.16b - eor v23.16b, v23.16b, v27.16b - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - mov v29.s[0], w26 + add V20.4S, V0.4S, V16.4S + add V21.4S, V4.4S, V17.4S + add V22.4S, V8.4S, V18.4S + add V23.4S, V12.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V1.4S, V16.4S + add V21.4S, V5.4S, V17.4S + add V22.4S, V9.4S, V18.4S + add V23.4S, V13.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V2.4S, V16.4S + add V21.4S, V6.4S, V17.4S + add V22.4S, V10.4S, V18.4S + add V23.4S, V14.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V20.4S, V3.4S, V16.4S + add V21.4S, V7.4S, V17.4S + add V22.4S, V11.4S, V18.4S + add V23.4S, V15.4S, V19.4S + eor V20.16B, V20.16B, V24.16B + eor V21.16B, V21.16B, V25.16B + eor V22.16B, V22.16B, V26.16B + eor V23.16B, V23.16B, V27.16B + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + mov V29.S[0], w26 sub x3, x3, #0x100 - add v19.4s, v19.4s, v29.4s + add V19.4S, V19.4S, V29.4S ; Done 256-byte block L_chacha_crypt_bytes_arm64_lt_256 cmp x3, #0x80 blt L_chacha_crypt_bytes_arm64_lt_128 - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 ; Move state into vector registers - mov v4.16b, v16.16b - mov v5.16b, v17.16b - mov v6.16b, v18.16b - mov v7.16b, v19.16b - mov v0.16b, v16.16b - mov v1.16b, v17.16b - mov v2.16b, v18.16b - mov v3.16b, v19.16b + mov V4.16B, V16.16B + mov V5.16B, V17.16B + mov V6.16B, V18.16B + mov V7.16B, V19.16B + mov V0.16B, V16.16B + mov V1.16B, V17.16B + mov V2.16B, V18.16B + mov V3.16B, V19.16B ; Add counter word - add v7.4s, v7.4s, v31.4s + add V7.4S, V7.4S, V31.4S ; Set number of odd+even rounds to perform mov x26, #10 L_chacha_crypt_bytes_arm64_round_start_128 subs x26, x26, #1 ; Round odd ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v1.4s - add v4.4s, v4.4s, v5.4s - eor v3.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v4.16b - rev32 v3.8h, v3.8h - rev32 v7.8h, v7.8h + add V0.4S, V0.4S, V1.4S + add V4.4S, V4.4S, V5.4S + eor V3.16B, V3.16B, V0.16B + eor V7.16B, V7.16B, V4.16B + rev32 V3.8H, V3.8H + rev32 V7.8H, V7.8H ; c += d; b ^= c; b <<<= 12; - add v2.4s, v2.4s, v3.4s - add v6.4s, v6.4s, v7.4s - eor v20.16b, v1.16b, v2.16b - eor v21.16b, v5.16b, v6.16b - shl v1.4s, v20.4s, #12 - shl v5.4s, v21.4s, #12 - sri v1.4s, v20.4s, #20 - sri v5.4s, v21.4s, #20 + add V2.4S, V2.4S, V3.4S + add V6.4S, V6.4S, V7.4S + eor V20.16B, V1.16B, V2.16B + eor V21.16B, V5.16B, V6.16B + shl V1.4S, V20.4S, #12 + shl V5.4S, V21.4S, #12 + sri V1.4S, V20.4S, #20 + sri V5.4S, V21.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v1.4s - add v4.4s, v4.4s, v5.4s - eor v3.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v4.16b - tbl v3.16b, {v3.16b}, v30.16b - tbl v7.16b, {v7.16b}, v30.16b + add V0.4S, V0.4S, V1.4S + add V4.4S, V4.4S, V5.4S + eor V3.16B, V3.16B, V0.16B + eor V7.16B, V7.16B, V4.16B + tbl V3.16B, {V3.16B}, V30.16B + tbl V7.16B, {V7.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v2.4s, v2.4s, v3.4s - add v6.4s, v6.4s, v7.4s - eor v20.16b, v1.16b, v2.16b - eor v21.16b, v5.16b, v6.16b - shl v1.4s, v20.4s, #7 - shl v5.4s, v21.4s, #7 - sri v1.4s, v20.4s, #25 - sri v5.4s, v21.4s, #25 - ext v3.16b, v3.16b, v3.16b, #12 - ext v7.16b, v7.16b, v7.16b, #12 - ext v1.16b, v1.16b, v1.16b, #4 - ext v5.16b, v5.16b, v5.16b, #4 - ext v2.16b, v2.16b, v2.16b, #8 - ext v6.16b, v6.16b, v6.16b, #8 + add V2.4S, V2.4S, V3.4S + add V6.4S, V6.4S, V7.4S + eor V20.16B, V1.16B, V2.16B + eor V21.16B, V5.16B, V6.16B + shl V1.4S, V20.4S, #7 + shl V5.4S, V21.4S, #7 + sri V1.4S, V20.4S, #25 + sri V5.4S, V21.4S, #25 + ext8 V3.16B, V3.16B, V3.16B, #12 + ext8 V7.16B, V7.16B, V7.16B, #12 + ext8 V1.16B, V1.16B, V1.16B, #4 + ext8 V5.16B, V5.16B, V5.16B, #4 + ext8 V2.16B, V2.16B, V2.16B, #8 + ext8 V6.16B, V6.16B, V6.16B, #8 ; Round even ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v1.4s - add v4.4s, v4.4s, v5.4s - eor v3.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v4.16b - rev32 v3.8h, v3.8h - rev32 v7.8h, v7.8h + add V0.4S, V0.4S, V1.4S + add V4.4S, V4.4S, V5.4S + eor V3.16B, V3.16B, V0.16B + eor V7.16B, V7.16B, V4.16B + rev32 V3.8H, V3.8H + rev32 V7.8H, V7.8H ; c += d; b ^= c; b <<<= 12; - add v2.4s, v2.4s, v3.4s - add v6.4s, v6.4s, v7.4s - eor v20.16b, v1.16b, v2.16b - eor v21.16b, v5.16b, v6.16b - shl v1.4s, v20.4s, #12 - shl v5.4s, v21.4s, #12 - sri v1.4s, v20.4s, #20 - sri v5.4s, v21.4s, #20 + add V2.4S, V2.4S, V3.4S + add V6.4S, V6.4S, V7.4S + eor V20.16B, V1.16B, V2.16B + eor V21.16B, V5.16B, V6.16B + shl V1.4S, V20.4S, #12 + shl V5.4S, V21.4S, #12 + sri V1.4S, V20.4S, #20 + sri V5.4S, V21.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v1.4s - add v4.4s, v4.4s, v5.4s - eor v3.16b, v3.16b, v0.16b - eor v7.16b, v7.16b, v4.16b - tbl v3.16b, {v3.16b}, v30.16b - tbl v7.16b, {v7.16b}, v30.16b + add V0.4S, V0.4S, V1.4S + add V4.4S, V4.4S, V5.4S + eor V3.16B, V3.16B, V0.16B + eor V7.16B, V7.16B, V4.16B + tbl V3.16B, {V3.16B}, V30.16B + tbl V7.16B, {V7.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v2.4s, v2.4s, v3.4s - add v6.4s, v6.4s, v7.4s - eor v20.16b, v1.16b, v2.16b - eor v21.16b, v5.16b, v6.16b - shl v1.4s, v20.4s, #7 - shl v5.4s, v21.4s, #7 - sri v1.4s, v20.4s, #25 - sri v5.4s, v21.4s, #25 - ext v3.16b, v3.16b, v3.16b, #4 - ext v7.16b, v7.16b, v7.16b, #4 - ext v1.16b, v1.16b, v1.16b, #12 - ext v5.16b, v5.16b, v5.16b, #12 - ext v2.16b, v2.16b, v2.16b, #8 - ext v6.16b, v6.16b, v6.16b, #8 + add V2.4S, V2.4S, V3.4S + add V6.4S, V6.4S, V7.4S + eor V20.16B, V1.16B, V2.16B + eor V21.16B, V5.16B, V6.16B + shl V1.4S, V20.4S, #7 + shl V5.4S, V21.4S, #7 + sri V1.4S, V20.4S, #25 + sri V5.4S, V21.4S, #25 + ext8 V3.16B, V3.16B, V3.16B, #4 + ext8 V7.16B, V7.16B, V7.16B, #4 + ext8 V1.16B, V1.16B, V1.16B, #12 + ext8 V5.16B, V5.16B, V5.16B, #12 + ext8 V2.16B, V2.16B, V2.16B, #8 + ext8 V6.16B, V6.16B, V6.16B, #8 bne L_chacha_crypt_bytes_arm64_round_start_128 ; Add back state, XOR in message and store (load next block) - add v0.4s, v0.4s, v16.4s - add v1.4s, v1.4s, v17.4s - add v2.4s, v2.4s, v18.4s - add v3.4s, v3.4s, v19.4s - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - eor v26.16b, v26.16b, v2.16b - eor v27.16b, v27.16b, v3.16b - ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x2], #0x40 - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 - add v19.4s, v19.4s, v31.4s - add v4.4s, v4.4s, v16.4s - add v5.4s, v5.4s, v17.4s - add v6.4s, v6.4s, v18.4s - add v7.4s, v7.4s, v19.4s - eor v20.16b, v20.16b, v4.16b - eor v21.16b, v21.16b, v5.16b - eor v22.16b, v22.16b, v6.16b - eor v23.16b, v23.16b, v7.16b - st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [x1], #0x40 - add v19.4s, v19.4s, v31.4s + add V0.4S, V0.4S, V16.4S + add V1.4S, V1.4S, V17.4S + add V2.4S, V2.4S, V18.4S + add V3.4S, V3.4S, V19.4S + eor V24.16B, V24.16B, V0.16B + eor V25.16B, V25.16B, V1.16B + eor V26.16B, V26.16B, V2.16B + eor V27.16B, V27.16B, V3.16B + ld1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x2], #0x40 + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 + add V19.4S, V19.4S, V31.4S + add V4.4S, V4.4S, V16.4S + add V5.4S, V5.4S, V17.4S + add V6.4S, V6.4S, V18.4S + add V7.4S, V7.4S, V19.4S + eor V20.16B, V20.16B, V4.16B + eor V21.16B, V21.16B, V5.16B + eor V22.16B, V22.16B, V6.16B + eor V23.16B, V23.16B, V7.16B + st1 {V20.16B, V21.16B, V22.16B, V23.16B}, [x1], #0x40 + add V19.4S, V19.4S, V31.4S sub x3, x3, #0x80 ; Done 128-byte block L_chacha_crypt_bytes_arm64_lt_128 @@ -798,75 +798,75 @@ L_chacha_crypt_bytes_arm64_lt_128 mov w5, #0x40 L_chacha_crypt_bytes_arm64_loop_64 ; Move state into vector registers - mov v0.16b, v16.16b - mov v1.16b, v17.16b - mov v2.16b, v18.16b - mov v3.16b, v19.16b + mov V0.16B, V16.16B + mov V1.16B, V17.16B + mov V2.16B, V18.16B + mov V3.16B, V19.16B ; Set number of odd+even rounds to perform mov x26, #10 L_chacha_crypt_bytes_arm64_round_64 subs x26, x26, #1 ; Round odd ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v1.4s - eor v3.16b, v3.16b, v0.16b - rev32 v3.8h, v3.8h + add V0.4S, V0.4S, V1.4S + eor V3.16B, V3.16B, V0.16B + rev32 V3.8H, V3.8H ; c += d; b ^= c; b <<<= 12; - add v2.4s, v2.4s, v3.4s - eor v20.16b, v1.16b, v2.16b - shl v1.4s, v20.4s, #12 - sri v1.4s, v20.4s, #20 + add V2.4S, V2.4S, V3.4S + eor V20.16B, V1.16B, V2.16B + shl V1.4S, V20.4S, #12 + sri V1.4S, V20.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v1.4s - eor v3.16b, v3.16b, v0.16b - tbl v3.16b, {v3.16b}, v30.16b + add V0.4S, V0.4S, V1.4S + eor V3.16B, V3.16B, V0.16B + tbl V3.16B, {V3.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v2.4s, v2.4s, v3.4s - eor v20.16b, v1.16b, v2.16b - shl v1.4s, v20.4s, #7 - sri v1.4s, v20.4s, #25 - ext v3.16b, v3.16b, v3.16b, #12 - ext v1.16b, v1.16b, v1.16b, #4 - ext v2.16b, v2.16b, v2.16b, #8 + add V2.4S, V2.4S, V3.4S + eor V20.16B, V1.16B, V2.16B + shl V1.4S, V20.4S, #7 + sri V1.4S, V20.4S, #25 + ext8 V3.16B, V3.16B, V3.16B, #12 + ext8 V1.16B, V1.16B, V1.16B, #4 + ext8 V2.16B, V2.16B, V2.16B, #8 ; Round even ; a += b; d ^= a; d <<<= 16; - add v0.4s, v0.4s, v1.4s - eor v3.16b, v3.16b, v0.16b - rev32 v3.8h, v3.8h + add V0.4S, V0.4S, V1.4S + eor V3.16B, V3.16B, V0.16B + rev32 V3.8H, V3.8H ; c += d; b ^= c; b <<<= 12; - add v2.4s, v2.4s, v3.4s - eor v20.16b, v1.16b, v2.16b - shl v1.4s, v20.4s, #12 - sri v1.4s, v20.4s, #20 + add V2.4S, V2.4S, V3.4S + eor V20.16B, V1.16B, V2.16B + shl V1.4S, V20.4S, #12 + sri V1.4S, V20.4S, #20 ; a += b; d ^= a; d <<<= 8; - add v0.4s, v0.4s, v1.4s - eor v3.16b, v3.16b, v0.16b - tbl v3.16b, {v3.16b}, v30.16b + add V0.4S, V0.4S, V1.4S + eor V3.16B, V3.16B, V0.16B + tbl V3.16B, {V3.16B}, V30.16B ; c += d; b ^= c; b <<<= 7; - add v2.4s, v2.4s, v3.4s - eor v20.16b, v1.16b, v2.16b - shl v1.4s, v20.4s, #7 - sri v1.4s, v20.4s, #25 - ext v3.16b, v3.16b, v3.16b, #4 - ext v1.16b, v1.16b, v1.16b, #12 - ext v2.16b, v2.16b, v2.16b, #8 + add V2.4S, V2.4S, V3.4S + eor V20.16B, V1.16B, V2.16B + shl V1.4S, V20.4S, #7 + sri V1.4S, V20.4S, #25 + ext8 V3.16B, V3.16B, V3.16B, #4 + ext8 V1.16B, V1.16B, V1.16B, #12 + ext8 V2.16B, V2.16B, V2.16B, #8 bne L_chacha_crypt_bytes_arm64_round_64 ; Add back state - add v0.4s, v0.4s, v16.4s - add v1.4s, v1.4s, v17.4s - add v2.4s, v2.4s, v18.4s - add v3.4s, v3.4s, v19.4s + add V0.4S, V0.4S, V16.4S + add V1.4S, V1.4S, V17.4S + add V2.4S, V2.4S, V18.4S + add V3.4S, V3.4S, V19.4S ; Check if data is less than 64 bytes - store in over cmp x3, #0x40 - add v19.4s, v19.4s, v31.4s + add V19.4S, V19.4S, V31.4S blt L_chacha_crypt_bytes_arm64_lt_64 ; Encipher 64 bytes - ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #0x40 - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - eor v26.16b, v26.16b, v2.16b - eor v27.16b, v27.16b, v3.16b - st1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x1], #0x40 + ld1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x2], #0x40 + eor V24.16B, V24.16B, V0.16B + eor V25.16B, V25.16B, V1.16B + eor V26.16B, V26.16B, V2.16B + eor V27.16B, V27.16B, V3.16B + st1 {V24.16B, V25.16B, V26.16B, V27.16B}, [x1], #0x40 ; Check for more bytes to be enciphered subs x3, x3, #0x40 bne L_chacha_crypt_bytes_arm64_loop_64 @@ -875,41 +875,41 @@ L_chacha_crypt_bytes_arm64_lt_64 ; Calculate bytes left in block not used sub w5, w5, w3 ; Store encipher block in over for further operations and left - st1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x4] + st1 {V0.4S, V1.4S, V2.4S, V3.4S}, [x4] str w5, [x0, #64] ; Encipher 32 bytes cmp x3, #32 blt L_chacha_crypt_bytes_arm64_lt_32 - ld1 {v24.16b, v25.16b}, [x2], #32 - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - st1 {v24.16b, v25.16b}, [x1], #32 + ld1 {V24.16B, V25.16B}, [x2], #32 + eor V24.16B, V24.16B, V0.16B + eor V25.16B, V25.16B, V1.16B + st1 {V24.16B, V25.16B}, [x1], #32 subs x3, x3, #32 - mov v0.16b, v2.16b - mov v1.16b, v3.16b + mov V0.16B, V2.16B + mov V1.16B, V3.16B beq L_chacha_crypt_bytes_arm64_done L_chacha_crypt_bytes_arm64_lt_32 cmp x3, #16 blt L_chacha_crypt_bytes_arm64_lt_16 ; Encipher 16 bytes - ld1 {v24.16b}, [x2], #16 - eor v24.16b, v24.16b, v0.16b - st1 {v24.16b}, [x1], #16 + ld1 {V24.16B}, [x2], #16 + eor V24.16B, V24.16B, V0.16B + st1 {V24.16B}, [x1], #16 subs x3, x3, #16 - mov v0.16b, v1.16b + mov V0.16B, V1.16B beq L_chacha_crypt_bytes_arm64_done L_chacha_crypt_bytes_arm64_lt_16 cmp x3, #8 blt L_chacha_crypt_bytes_arm64_lt_8 ; Encipher 8 bytes - ld1 {v24.8b}, [x2], #8 - eor v24.8b, v24.8b, v0.8b - st1 {v24.8b}, [x1], #8 + ld1 {V24.8B}, [x2], #8 + eor V24.8B, V24.8B, V0.8B + st1 {V24.8B}, [x1], #8 subs x3, x3, #8 - mov v0.d[0], v0.d[1] + mov V0.D[0], V0.D[1] beq L_chacha_crypt_bytes_arm64_done L_chacha_crypt_bytes_arm64_lt_8 - mov x5, v0.d[0] + mov x5, V0.D[0] L_chacha_crypt_bytes_arm64_loop_lt_8 ; Encipher 1 byte at a time ldrb w6, [x2], #1 @@ -920,16 +920,16 @@ L_chacha_crypt_bytes_arm64_loop_lt_8 bgt L_chacha_crypt_bytes_arm64_loop_lt_8 L_chacha_crypt_bytes_arm64_done L_chacha_crypt_bytes_arm64_done_all - st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x0] + st1 {V16.4S, V17.4S, V18.4S, V19.4S}, [x0] ldp x17, x19, [x29, #24] ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldp x24, x25, [x29, #72] ldr x26, [x29, #88] - ldp d8, d9, [x29, #96] - ldp d10, d11, [x29, #112] - ldp d12, d13, [x29, #128] - ldp d14, d15, [x29, #144] + ldp D8, D9, [x29, #96] + ldp D10, D11, [x29, #112] + ldp D12, D13, [x29, #128] + ldp D14, D15, [x29, #144] ldp x29, x30, [sp], #0xa0 ret ENDP @@ -944,7 +944,7 @@ wc_chacha_setiv PROC str w4, [x0, #60] ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_chacha_setkey_arm64_constant DCD 0x61707865, 0x3120646e, 0x79622d36, 0x6b206574 @@ -958,20 +958,20 @@ wc_chacha_setkey PROC subs x2, x2, #16 add x3, x3, x2 ; Start with constants - ld1 {v0.4s}, [x3] - ld1 {v1.16b}, [x1], #16 + ld1 {V0.4S}, [x3] + ld1 {V1.16B}, [x1], #16 IF :DEF:BIG_ENDIAN_ORDER - rev32 v1.8h, v1.8h + rev32 V1.8H, V1.8H ENDIF - st1 {v0.4s}, [x0], #16 - st1 {v1.4s}, [x0], #16 + st1 {V0.4S}, [x0], #16 + st1 {V1.4S}, [x0], #16 beq L_chacha_setkey_arm64_done - ld1 {v1.16b}, [x1] + ld1 {V1.16B}, [x1] IF :DEF:BIG_ENDIAN_ORDER - rev32 v1.8h, v1.8h + rev32 V1.8H, V1.8H ENDIF L_chacha_setkey_arm64_done - st1 {v1.4s}, [x0] + st1 {V1.4S}, [x0] ret ENDP AREA |.text|, CODE, READONLY @@ -982,11 +982,11 @@ L_chacha_use_over_arm64_16byte_loop cmp x3, #16 blt L_chacha_use_over_arm64_word_loop ; 16 bytes of state XORed into message. - ld1 {v0.16b}, [x0], #16 - ld1 {v1.16b}, [x2], #16 - eor v1.16b, v1.16b, v0.16b + ld1 {V0.16B}, [x0], #16 + ld1 {V1.16B}, [x2], #16 + eor V1.16B, V1.16B, V0.16B subs x3, x3, #16 - st1 {v1.16b}, [x1], #16 + st1 {V1.16B}, [x1], #16 beq L_chacha_use_over_arm64_done b L_chacha_use_over_arm64_16byte_loop L_chacha_use_over_arm64_word_loop diff --git a/wolfcrypt/src/port/arm/armv8-curve25519.asm b/wolfcrypt/src/port/arm/armv8-curve25519.asm index 6c06c06bb6..e2706c3614 100644 --- a/wolfcrypt/src/port/arm/armv8-curve25519.asm +++ b/wolfcrypt/src/port/arm/armv8-curve25519.asm @@ -1696,7 +1696,7 @@ L_fe_invert8 ret ENDP IF :LNOT::DEF:HAVE_ED25519 :LAND: :LNOT::DEF:WOLFSSL_CURVE25519_USE_ED25519 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_curve25519_base_x2 DCQ 0x5cae469cdd684efb, 0x8f3f5ced1e350b5c @@ -5047,6 +5047,14 @@ L_curve25519_bits adcs x7, x7, x25 adcs x8, x8, x26 adc x9, x9, x27 + ; Reduce if top bit set + mov x3, #19 + and x4, x3, x9, asr 63 + adds x6, x6, x4 + adcs x7, x7, xzr + and x9, x9, #0x7fffffffffffffff + adcs x8, x8, xzr + adc x9, x9, xzr ; Square ; A[0] * A[1] umulh x16, x19, x20 diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm index cc8271d85e..cc93e3147d 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm @@ -24,12 +24,12 @@ ; cd ../scripts ; ruby ./kyber/kyber.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_consts DCW 0x0d01, 0xf301, 0x4ebf, 0x0549, 0x5049, 0x0000, 0x0000, 0x0000 IF :DEF:WOLFSSL_HAVE_MLKEM - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_zetas DCW 0x08ed, 0x0a0b, 0x0b9a, 0x0714, 0x05d5, 0x058e, 0x011f, 0x00ca @@ -68,7 +68,7 @@ L_mlkem_aarch64_zetas DCW 0x071b, 0x071b, 0x09ab, 0x09ab, 0x099b, 0x099b, 0x01de, 0x01de DCW 0x0c95, 0x0c95, 0x0bcd, 0x0bcd, 0x03e4, 0x03e4, 0x03df, 0x03df DCW 0x03be, 0x03be, 0x074d, 0x074d, 0x05f2, 0x05f2, 0x065c, 0x065c - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_zetas_qinv DCW 0xffed, 0x7b0b, 0x399a, 0x0314, 0x34d5, 0xcf8e, 0x6e1f, 0xbeca @@ -113,10 +113,10 @@ L_mlkem_aarch64_zetas_qinv mlkem_ntt PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_zetas add x2, x2, L_mlkem_aarch64_zetas adrp x3, L_mlkem_aarch64_zetas_qinv @@ -124,1259 +124,1259 @@ mlkem_ntt PROC adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts add x1, x0, #0x100 - ldr q4, [x4] - ldr q5, [x0] - ldr q6, [x0, #32] - ldr q7, [x0, #64] - ldr q8, [x0, #96] - ldr q9, [x0, #128] - ldr q10, [x0, #160] - ldr q11, [x0, #192] - ldr q12, [x0, #224] - ldr q13, [x1] - ldr q14, [x1, #32] - ldr q15, [x1, #64] - ldr q16, [x1, #96] - ldr q17, [x1, #128] - ldr q18, [x1, #160] - ldr q19, [x1, #192] - ldr q20, [x1, #224] - ldr q0, [x2] - ldr q1, [x3] - mul v29.8h, v13.8h, v1.h[1] - mul v30.8h, v14.8h, v1.h[1] - sqrdmulh v21.8h, v13.8h, v0.h[1] - sqrdmulh v22.8h, v14.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v15.8h, v1.h[1] - mul v30.8h, v16.8h, v1.h[1] - sqrdmulh v23.8h, v15.8h, v0.h[1] - sqrdmulh v24.8h, v16.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[1] - mul v30.8h, v18.8h, v1.h[1] - sqrdmulh v25.8h, v17.8h, v0.h[1] - sqrdmulh v26.8h, v18.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[1] - mul v30.8h, v20.8h, v1.h[1] - sqrdmulh v27.8h, v19.8h, v0.h[1] - sqrdmulh v28.8h, v20.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v13.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v14.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v15.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v16.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v9.8h, v25.8h - add v9.8h, v9.8h, v25.8h - sub v18.8h, v10.8h, v26.8h - add v10.8h, v10.8h, v26.8h - sub v19.8h, v11.8h, v27.8h - add v11.8h, v11.8h, v27.8h - sub v20.8h, v12.8h, v28.8h - add v12.8h, v12.8h, v28.8h - mul v29.8h, v9.8h, v1.h[2] - mul v30.8h, v10.8h, v1.h[2] - sqrdmulh v21.8h, v9.8h, v0.h[2] - sqrdmulh v22.8h, v10.8h, v0.h[2] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[2] - sqrdmulh v23.8h, v11.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[2] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[3] - mul v30.8h, v18.8h, v1.h[3] - sqrdmulh v25.8h, v17.8h, v0.h[3] - sqrdmulh v26.8h, v18.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[3] - mul v30.8h, v20.8h, v1.h[3] - sqrdmulh v27.8h, v19.8h, v0.h[3] - sqrdmulh v28.8h, v20.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v9.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v10.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v12.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v18.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v15.8h, v27.8h - add v15.8h, v15.8h, v27.8h - sub v20.8h, v16.8h, v28.8h - add v16.8h, v16.8h, v28.8h - mul v29.8h, v7.8h, v1.h[4] - mul v30.8h, v8.8h, v1.h[4] - sqrdmulh v21.8h, v7.8h, v0.h[4] - sqrdmulh v22.8h, v8.8h, v0.h[4] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[5] - mul v30.8h, v12.8h, v1.h[5] - sqrdmulh v23.8h, v11.8h, v0.h[5] - sqrdmulh v24.8h, v12.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v15.8h, v1.h[6] - mul v30.8h, v16.8h, v1.h[6] - sqrdmulh v25.8h, v15.8h, v0.h[6] - sqrdmulh v26.8h, v16.8h, v0.h[6] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[7] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v19.8h, v0.h[7] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v7.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v10.8h, v24.8h - add v10.8h, v10.8h, v24.8h - sub v15.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v18.8h, v28.8h - add v18.8h, v18.8h, v28.8h - ldr q0, [x2, #16] - ldr q1, [x3, #16] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - str q5, [x0] - str q6, [x0, #32] - str q7, [x0, #64] - str q8, [x0, #96] - str q9, [x0, #128] - str q10, [x0, #160] - str q11, [x0, #192] - str q12, [x0, #224] - str q13, [x1] - str q14, [x1, #32] - str q15, [x1, #64] - str q16, [x1, #96] - str q17, [x1, #128] - str q18, [x1, #160] - str q19, [x1, #192] - str q20, [x1, #224] - ldr q5, [x0, #16] - ldr q6, [x0, #48] - ldr q7, [x0, #80] - ldr q8, [x0, #112] - ldr q9, [x0, #144] - ldr q10, [x0, #176] - ldr q11, [x0, #208] - ldr q12, [x0, #240] - ldr q13, [x1, #16] - ldr q14, [x1, #48] - ldr q15, [x1, #80] - ldr q16, [x1, #112] - ldr q17, [x1, #144] - ldr q18, [x1, #176] - ldr q19, [x1, #208] - ldr q20, [x1, #240] - ldr q0, [x2] - ldr q1, [x3] - mul v29.8h, v13.8h, v1.h[1] - mul v30.8h, v14.8h, v1.h[1] - sqrdmulh v21.8h, v13.8h, v0.h[1] - sqrdmulh v22.8h, v14.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v15.8h, v1.h[1] - mul v30.8h, v16.8h, v1.h[1] - sqrdmulh v23.8h, v15.8h, v0.h[1] - sqrdmulh v24.8h, v16.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[1] - mul v30.8h, v18.8h, v1.h[1] - sqrdmulh v25.8h, v17.8h, v0.h[1] - sqrdmulh v26.8h, v18.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[1] - mul v30.8h, v20.8h, v1.h[1] - sqrdmulh v27.8h, v19.8h, v0.h[1] - sqrdmulh v28.8h, v20.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v13.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v14.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v15.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v16.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v9.8h, v25.8h - add v9.8h, v9.8h, v25.8h - sub v18.8h, v10.8h, v26.8h - add v10.8h, v10.8h, v26.8h - sub v19.8h, v11.8h, v27.8h - add v11.8h, v11.8h, v27.8h - sub v20.8h, v12.8h, v28.8h - add v12.8h, v12.8h, v28.8h - mul v29.8h, v9.8h, v1.h[2] - mul v30.8h, v10.8h, v1.h[2] - sqrdmulh v21.8h, v9.8h, v0.h[2] - sqrdmulh v22.8h, v10.8h, v0.h[2] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[2] - sqrdmulh v23.8h, v11.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[2] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[3] - mul v30.8h, v18.8h, v1.h[3] - sqrdmulh v25.8h, v17.8h, v0.h[3] - sqrdmulh v26.8h, v18.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[3] - mul v30.8h, v20.8h, v1.h[3] - sqrdmulh v27.8h, v19.8h, v0.h[3] - sqrdmulh v28.8h, v20.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v9.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v10.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v12.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v18.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v15.8h, v27.8h - add v15.8h, v15.8h, v27.8h - sub v20.8h, v16.8h, v28.8h - add v16.8h, v16.8h, v28.8h - mul v29.8h, v7.8h, v1.h[4] - mul v30.8h, v8.8h, v1.h[4] - sqrdmulh v21.8h, v7.8h, v0.h[4] - sqrdmulh v22.8h, v8.8h, v0.h[4] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[5] - mul v30.8h, v12.8h, v1.h[5] - sqrdmulh v23.8h, v11.8h, v0.h[5] - sqrdmulh v24.8h, v12.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v15.8h, v1.h[6] - mul v30.8h, v16.8h, v1.h[6] - sqrdmulh v25.8h, v15.8h, v0.h[6] - sqrdmulh v26.8h, v16.8h, v0.h[6] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[7] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v19.8h, v0.h[7] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v7.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v10.8h, v24.8h - add v10.8h, v10.8h, v24.8h - sub v15.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v18.8h, v28.8h - add v18.8h, v18.8h, v28.8h - ldr q0, [x2, #16] - ldr q1, [x3, #16] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - str q5, [x0, #16] - str q6, [x0, #48] - str q7, [x0, #80] - str q8, [x0, #112] - str q9, [x0, #144] - str q10, [x0, #176] - str q11, [x0, #208] - str q12, [x0, #240] - str q13, [x1, #16] - str q14, [x1, #48] - str q15, [x1, #80] - str q16, [x1, #112] - str q17, [x1, #144] - str q18, [x1, #176] - str q19, [x1, #208] - str q20, [x1, #240] - ldp q5, q6, [x0] - ldp q7, q8, [x0, #32] - ldp q9, q10, [x0, #64] - ldp q11, q12, [x0, #96] - ldp q13, q14, [x0, #128] - ldp q15, q16, [x0, #160] - ldp q17, q18, [x0, #192] - ldp q19, q20, [x0, #224] - ldr q0, [x2, #32] - ldr q1, [x3, #32] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #64] - ldr q2, [x2, #80] - ldr q1, [x3, #64] - ldr q3, [x3, #80] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.2d, v5.2d, v6.2d - trn1 v7.2d, v7.2d, v8.2d - trn2 v6.2d, v29.2d, v6.2d - trn2 v8.2d, v30.2d, v8.2d - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #96] - ldr q2, [x2, #112] - ldr q1, [x3, #96] - ldr q3, [x3, #112] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v29.2d, v10.2d - trn2 v12.2d, v30.2d, v12.2d - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #128] - ldr q2, [x2, #144] - ldr q1, [x3, #128] - ldr q3, [x3, #144] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v29.2d, v14.2d - trn2 v16.2d, v30.2d, v16.2d - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #160] - ldr q2, [x2, #176] - ldr q1, [x3, #160] - ldr q3, [x3, #176] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v29.2d, v18.2d - trn2 v20.2d, v30.2d, v20.2d - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #320] - ldr q2, [x2, #336] - ldr q1, [x3, #320] - ldr q3, [x3, #336] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.4s, v5.4s, v6.4s - trn1 v7.4s, v7.4s, v8.4s - trn2 v6.4s, v29.4s, v6.4s - trn2 v8.4s, v30.4s, v8.4s - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #352] - ldr q2, [x2, #368] - ldr q1, [x3, #352] - ldr q3, [x3, #368] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v29.4s, v10.4s - trn2 v12.4s, v30.4s, v12.4s - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #384] - ldr q2, [x2, #400] - ldr q1, [x3, #384] - ldr q3, [x3, #400] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v29.4s, v14.4s - trn2 v16.4s, v30.4s, v16.4s - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #416] - ldr q2, [x2, #432] - ldr q1, [x3, #416] - ldr q3, [x3, #432] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v29.4s, v18.4s - trn2 v20.4s, v30.4s, v20.4s - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - sqdmulh v21.8h, v5.8h, v4.h[2] - sqdmulh v22.8h, v6.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v5.8h, v21.8h, v4.h[0] - mls v6.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v7.8h, v4.h[2] - sqdmulh v22.8h, v8.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v7.8h, v21.8h, v4.h[0] - mls v8.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v9.8h, v4.h[2] - sqdmulh v22.8h, v10.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v9.8h, v21.8h, v4.h[0] - mls v10.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v11.8h, v4.h[2] - sqdmulh v22.8h, v12.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v11.8h, v21.8h, v4.h[0] - mls v12.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v13.8h, v4.h[2] - sqdmulh v22.8h, v14.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v13.8h, v21.8h, v4.h[0] - mls v14.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v15.8h, v4.h[2] - sqdmulh v22.8h, v16.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v15.8h, v21.8h, v4.h[0] - mls v16.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v17.8h, v4.h[2] - sqdmulh v22.8h, v18.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v17.8h, v21.8h, v4.h[0] - mls v18.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v19.8h, v4.h[2] - sqdmulh v22.8h, v20.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v19.8h, v21.8h, v4.h[0] - mls v20.8h, v22.8h, v4.h[0] - mov v29.16b, v5.16b - trn1 v5.4s, v5.4s, v6.4s - trn2 v6.4s, v29.4s, v6.4s - mov v29.16b, v5.16b - trn1 v5.2d, v5.2d, v6.2d - trn2 v6.2d, v29.2d, v6.2d - mov v29.16b, v7.16b - trn1 v7.4s, v7.4s, v8.4s - trn2 v8.4s, v29.4s, v8.4s - mov v29.16b, v7.16b - trn1 v7.2d, v7.2d, v8.2d - trn2 v8.2d, v29.2d, v8.2d - mov v29.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v29.4s, v10.4s - mov v29.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v29.2d, v10.2d - mov v29.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v29.4s, v12.4s - mov v29.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v29.2d, v12.2d - mov v29.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v29.4s, v14.4s - mov v29.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v29.2d, v14.2d - mov v29.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v29.4s, v16.4s - mov v29.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v29.2d, v16.2d - mov v29.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v29.4s, v18.4s - mov v29.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v29.2d, v18.2d - mov v29.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v29.4s, v20.4s - mov v29.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v29.2d, v20.2d - stp q5, q6, [x0] - stp q7, q8, [x0, #32] - stp q9, q10, [x0, #64] - stp q11, q12, [x0, #96] - stp q13, q14, [x0, #128] - stp q15, q16, [x0, #160] - stp q17, q18, [x0, #192] - stp q19, q20, [x0, #224] - ldp q5, q6, [x1] - ldp q7, q8, [x1, #32] - ldp q9, q10, [x1, #64] - ldp q11, q12, [x1, #96] - ldp q13, q14, [x1, #128] - ldp q15, q16, [x1, #160] - ldp q17, q18, [x1, #192] - ldp q19, q20, [x1, #224] - ldr q0, [x2, #48] - ldr q1, [x3, #48] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #192] - ldr q2, [x2, #208] - ldr q1, [x3, #192] - ldr q3, [x3, #208] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.2d, v5.2d, v6.2d - trn1 v7.2d, v7.2d, v8.2d - trn2 v6.2d, v29.2d, v6.2d - trn2 v8.2d, v30.2d, v8.2d - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #224] - ldr q2, [x2, #240] - ldr q1, [x3, #224] - ldr q3, [x3, #240] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v29.2d, v10.2d - trn2 v12.2d, v30.2d, v12.2d - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #256] - ldr q2, [x2, #272] - ldr q1, [x3, #256] - ldr q3, [x3, #272] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v29.2d, v14.2d - trn2 v16.2d, v30.2d, v16.2d - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #288] - ldr q2, [x2, #304] - ldr q1, [x3, #288] - ldr q3, [x3, #304] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v29.2d, v18.2d - trn2 v20.2d, v30.2d, v20.2d - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #448] - ldr q2, [x2, #464] - ldr q1, [x3, #448] - ldr q3, [x3, #464] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.4s, v5.4s, v6.4s - trn1 v7.4s, v7.4s, v8.4s - trn2 v6.4s, v29.4s, v6.4s - trn2 v8.4s, v30.4s, v8.4s - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v21.8h, v21.8h, v29.8h - sub v22.8h, v22.8h, v30.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #480] - ldr q2, [x2, #496] - ldr q1, [x3, #480] - ldr q3, [x3, #496] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v29.4s, v10.4s - trn2 v12.4s, v30.4s, v12.4s - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v23.8h, v23.8h, v29.8h - sub v24.8h, v24.8h, v30.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #512] - ldr q2, [x2, #528] - ldr q1, [x3, #512] - ldr q3, [x3, #528] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v29.4s, v14.4s - trn2 v16.4s, v30.4s, v16.4s - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v25.8h, v25.8h, v29.8h - sub v26.8h, v26.8h, v30.8h - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #544] - ldr q2, [x2, #560] - ldr q1, [x3, #544] - ldr q3, [x3, #560] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v29.4s, v18.4s - trn2 v20.4s, v30.4s, v20.4s - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmulh v29.8h, v29.8h, v4.h[0] - sqrdmulh v30.8h, v30.8h, v4.h[0] - sub v27.8h, v27.8h, v29.8h - sub v28.8h, v28.8h, v30.8h - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - sqdmulh v21.8h, v5.8h, v4.h[2] - sqdmulh v22.8h, v6.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v5.8h, v21.8h, v4.h[0] - mls v6.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v7.8h, v4.h[2] - sqdmulh v22.8h, v8.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v7.8h, v21.8h, v4.h[0] - mls v8.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v9.8h, v4.h[2] - sqdmulh v22.8h, v10.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v9.8h, v21.8h, v4.h[0] - mls v10.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v11.8h, v4.h[2] - sqdmulh v22.8h, v12.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v11.8h, v21.8h, v4.h[0] - mls v12.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v13.8h, v4.h[2] - sqdmulh v22.8h, v14.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v13.8h, v21.8h, v4.h[0] - mls v14.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v15.8h, v4.h[2] - sqdmulh v22.8h, v16.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v15.8h, v21.8h, v4.h[0] - mls v16.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v17.8h, v4.h[2] - sqdmulh v22.8h, v18.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v17.8h, v21.8h, v4.h[0] - mls v18.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v19.8h, v4.h[2] - sqdmulh v22.8h, v20.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v19.8h, v21.8h, v4.h[0] - mls v20.8h, v22.8h, v4.h[0] - mov v29.16b, v5.16b - trn1 v5.4s, v5.4s, v6.4s - trn2 v6.4s, v29.4s, v6.4s - mov v29.16b, v5.16b - trn1 v5.2d, v5.2d, v6.2d - trn2 v6.2d, v29.2d, v6.2d - mov v29.16b, v7.16b - trn1 v7.4s, v7.4s, v8.4s - trn2 v8.4s, v29.4s, v8.4s - mov v29.16b, v7.16b - trn1 v7.2d, v7.2d, v8.2d - trn2 v8.2d, v29.2d, v8.2d - mov v29.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v29.4s, v10.4s - mov v29.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v29.2d, v10.2d - mov v29.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v29.4s, v12.4s - mov v29.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v29.2d, v12.2d - mov v29.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v29.4s, v14.4s - mov v29.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v29.2d, v14.2d - mov v29.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v29.4s, v16.4s - mov v29.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v29.2d, v16.2d - mov v29.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v29.4s, v18.4s - mov v29.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v29.2d, v18.2d - mov v29.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v29.4s, v20.4s - mov v29.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v29.2d, v20.2d - stp q5, q6, [x1] - stp q7, q8, [x1, #32] - stp q9, q10, [x1, #64] - stp q11, q12, [x1, #96] - stp q13, q14, [x1, #128] - stp q15, q16, [x1, #160] - stp q17, q18, [x1, #192] - stp q19, q20, [x1, #224] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q4, [x4] + ldr Q5, [x0] + ldr Q6, [x0, #32] + ldr Q7, [x0, #64] + ldr Q8, [x0, #96] + ldr Q9, [x0, #128] + ldr Q10, [x0, #160] + ldr Q11, [x0, #192] + ldr Q12, [x0, #224] + ldr Q13, [x1] + ldr Q14, [x1, #32] + ldr Q15, [x1, #64] + ldr Q16, [x1, #96] + ldr Q17, [x1, #128] + ldr Q18, [x1, #160] + ldr Q19, [x1, #192] + ldr Q20, [x1, #224] + ldr Q0, [x2] + ldr Q1, [x3] + mul V29.8H, V13.8H, V1.H[1] + mul V30.8H, V14.8H, V1.H[1] + sqrdmulh V21.8H, V13.8H, V0.H[1] + sqrdmulh V22.8H, V14.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V15.8H, V1.H[1] + mul V30.8H, V16.8H, V1.H[1] + sqrdmulh V23.8H, V15.8H, V0.H[1] + sqrdmulh V24.8H, V16.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[1] + mul V30.8H, V18.8H, V1.H[1] + sqrdmulh V25.8H, V17.8H, V0.H[1] + sqrdmulh V26.8H, V18.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[1] + mul V30.8H, V20.8H, V1.H[1] + sqrdmulh V27.8H, V19.8H, V0.H[1] + sqrdmulh V28.8H, V20.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V13.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V14.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V15.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V16.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V9.8H, V25.8H + add V9.8H, V9.8H, V25.8H + sub V18.8H, V10.8H, V26.8H + add V10.8H, V10.8H, V26.8H + sub V19.8H, V11.8H, V27.8H + add V11.8H, V11.8H, V27.8H + sub V20.8H, V12.8H, V28.8H + add V12.8H, V12.8H, V28.8H + mul V29.8H, V9.8H, V1.H[2] + mul V30.8H, V10.8H, V1.H[2] + sqrdmulh V21.8H, V9.8H, V0.H[2] + sqrdmulh V22.8H, V10.8H, V0.H[2] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[2] + sqrdmulh V23.8H, V11.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[2] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[3] + mul V30.8H, V18.8H, V1.H[3] + sqrdmulh V25.8H, V17.8H, V0.H[3] + sqrdmulh V26.8H, V18.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[3] + mul V30.8H, V20.8H, V1.H[3] + sqrdmulh V27.8H, V19.8H, V0.H[3] + sqrdmulh V28.8H, V20.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V9.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V10.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V12.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V18.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V15.8H, V27.8H + add V15.8H, V15.8H, V27.8H + sub V20.8H, V16.8H, V28.8H + add V16.8H, V16.8H, V28.8H + mul V29.8H, V7.8H, V1.H[4] + mul V30.8H, V8.8H, V1.H[4] + sqrdmulh V21.8H, V7.8H, V0.H[4] + sqrdmulh V22.8H, V8.8H, V0.H[4] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[5] + mul V30.8H, V12.8H, V1.H[5] + sqrdmulh V23.8H, V11.8H, V0.H[5] + sqrdmulh V24.8H, V12.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V15.8H, V1.H[6] + mul V30.8H, V16.8H, V1.H[6] + sqrdmulh V25.8H, V15.8H, V0.H[6] + sqrdmulh V26.8H, V16.8H, V0.H[6] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[7] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V19.8H, V0.H[7] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V7.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V10.8H, V24.8H + add V10.8H, V10.8H, V24.8H + sub V15.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V18.8H, V28.8H + add V18.8H, V18.8H, V28.8H + ldr Q0, [x2, #16] + ldr Q1, [x3, #16] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + str Q5, [x0] + str Q6, [x0, #32] + str Q7, [x0, #64] + str Q8, [x0, #96] + str Q9, [x0, #128] + str Q10, [x0, #160] + str Q11, [x0, #192] + str Q12, [x0, #224] + str Q13, [x1] + str Q14, [x1, #32] + str Q15, [x1, #64] + str Q16, [x1, #96] + str Q17, [x1, #128] + str Q18, [x1, #160] + str Q19, [x1, #192] + str Q20, [x1, #224] + ldr Q5, [x0, #16] + ldr Q6, [x0, #48] + ldr Q7, [x0, #80] + ldr Q8, [x0, #112] + ldr Q9, [x0, #144] + ldr Q10, [x0, #176] + ldr Q11, [x0, #208] + ldr Q12, [x0, #240] + ldr Q13, [x1, #16] + ldr Q14, [x1, #48] + ldr Q15, [x1, #80] + ldr Q16, [x1, #112] + ldr Q17, [x1, #144] + ldr Q18, [x1, #176] + ldr Q19, [x1, #208] + ldr Q20, [x1, #240] + ldr Q0, [x2] + ldr Q1, [x3] + mul V29.8H, V13.8H, V1.H[1] + mul V30.8H, V14.8H, V1.H[1] + sqrdmulh V21.8H, V13.8H, V0.H[1] + sqrdmulh V22.8H, V14.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V15.8H, V1.H[1] + mul V30.8H, V16.8H, V1.H[1] + sqrdmulh V23.8H, V15.8H, V0.H[1] + sqrdmulh V24.8H, V16.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[1] + mul V30.8H, V18.8H, V1.H[1] + sqrdmulh V25.8H, V17.8H, V0.H[1] + sqrdmulh V26.8H, V18.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[1] + mul V30.8H, V20.8H, V1.H[1] + sqrdmulh V27.8H, V19.8H, V0.H[1] + sqrdmulh V28.8H, V20.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V13.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V14.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V15.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V16.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V9.8H, V25.8H + add V9.8H, V9.8H, V25.8H + sub V18.8H, V10.8H, V26.8H + add V10.8H, V10.8H, V26.8H + sub V19.8H, V11.8H, V27.8H + add V11.8H, V11.8H, V27.8H + sub V20.8H, V12.8H, V28.8H + add V12.8H, V12.8H, V28.8H + mul V29.8H, V9.8H, V1.H[2] + mul V30.8H, V10.8H, V1.H[2] + sqrdmulh V21.8H, V9.8H, V0.H[2] + sqrdmulh V22.8H, V10.8H, V0.H[2] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[2] + sqrdmulh V23.8H, V11.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[2] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[3] + mul V30.8H, V18.8H, V1.H[3] + sqrdmulh V25.8H, V17.8H, V0.H[3] + sqrdmulh V26.8H, V18.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[3] + mul V30.8H, V20.8H, V1.H[3] + sqrdmulh V27.8H, V19.8H, V0.H[3] + sqrdmulh V28.8H, V20.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V9.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V10.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V12.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V18.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V15.8H, V27.8H + add V15.8H, V15.8H, V27.8H + sub V20.8H, V16.8H, V28.8H + add V16.8H, V16.8H, V28.8H + mul V29.8H, V7.8H, V1.H[4] + mul V30.8H, V8.8H, V1.H[4] + sqrdmulh V21.8H, V7.8H, V0.H[4] + sqrdmulh V22.8H, V8.8H, V0.H[4] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[5] + mul V30.8H, V12.8H, V1.H[5] + sqrdmulh V23.8H, V11.8H, V0.H[5] + sqrdmulh V24.8H, V12.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V15.8H, V1.H[6] + mul V30.8H, V16.8H, V1.H[6] + sqrdmulh V25.8H, V15.8H, V0.H[6] + sqrdmulh V26.8H, V16.8H, V0.H[6] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[7] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V19.8H, V0.H[7] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V7.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V10.8H, V24.8H + add V10.8H, V10.8H, V24.8H + sub V15.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V18.8H, V28.8H + add V18.8H, V18.8H, V28.8H + ldr Q0, [x2, #16] + ldr Q1, [x3, #16] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + str Q5, [x0, #16] + str Q6, [x0, #48] + str Q7, [x0, #80] + str Q8, [x0, #112] + str Q9, [x0, #144] + str Q10, [x0, #176] + str Q11, [x0, #208] + str Q12, [x0, #240] + str Q13, [x1, #16] + str Q14, [x1, #48] + str Q15, [x1, #80] + str Q16, [x1, #112] + str Q17, [x1, #144] + str Q18, [x1, #176] + str Q19, [x1, #208] + str Q20, [x1, #240] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x2, #32] + ldr Q1, [x3, #32] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #64] + ldr Q2, [x2, #80] + ldr Q1, [x3, #64] + ldr Q3, [x3, #80] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #96] + ldr Q2, [x2, #112] + ldr Q1, [x3, #96] + ldr Q3, [x3, #112] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #128] + ldr Q2, [x2, #144] + ldr Q1, [x3, #128] + ldr Q3, [x3, #144] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #160] + ldr Q2, [x2, #176] + ldr Q1, [x3, #160] + ldr Q3, [x3, #176] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #320] + ldr Q2, [x2, #336] + ldr Q1, [x3, #320] + ldr Q3, [x3, #336] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #352] + ldr Q2, [x2, #368] + ldr Q1, [x3, #352] + ldr Q3, [x3, #368] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #384] + ldr Q2, [x2, #400] + ldr Q1, [x3, #384] + ldr Q3, [x3, #400] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #416] + ldr Q2, [x2, #432] + ldr Q1, [x3, #416] + ldr Q3, [x3, #432] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + sqdmulh V21.8H, V5.8H, V4.H[2] + sqdmulh V22.8H, V6.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V5.8H, V21.8H, V4.H[0] + mls V6.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V7.8H, V4.H[2] + sqdmulh V22.8H, V8.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V7.8H, V21.8H, V4.H[0] + mls V8.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V9.8H, V4.H[2] + sqdmulh V22.8H, V10.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V9.8H, V21.8H, V4.H[0] + mls V10.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V11.8H, V4.H[2] + sqdmulh V22.8H, V12.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V11.8H, V21.8H, V4.H[0] + mls V12.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V13.8H, V4.H[2] + sqdmulh V22.8H, V14.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V13.8H, V21.8H, V4.H[0] + mls V14.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V15.8H, V4.H[2] + sqdmulh V22.8H, V16.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V15.8H, V21.8H, V4.H[0] + mls V16.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V17.8H, V4.H[2] + sqdmulh V22.8H, V18.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V17.8H, V21.8H, V4.H[0] + mls V18.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V19.8H, V4.H[2] + sqdmulh V22.8H, V20.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V19.8H, V21.8H, V4.H[0] + mls V20.8H, V22.8H, V4.H[0] + mov V29.16B, V5.16B + trn1 V5.4S, V5.4S, V6.4S + trn2 V6.4S, V29.4S, V6.4S + mov V29.16B, V5.16B + trn1 V5.2D, V5.2D, V6.2D + trn2 V6.2D, V29.2D, V6.2D + mov V29.16B, V7.16B + trn1 V7.4S, V7.4S, V8.4S + trn2 V8.4S, V29.4S, V8.4S + mov V29.16B, V7.16B + trn1 V7.2D, V7.2D, V8.2D + trn2 V8.2D, V29.2D, V8.2D + mov V29.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V29.4S, V10.4S + mov V29.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V29.2D, V10.2D + mov V29.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V29.4S, V12.4S + mov V29.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V29.2D, V12.2D + mov V29.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V29.4S, V14.4S + mov V29.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V29.2D, V14.2D + mov V29.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V29.4S, V16.4S + mov V29.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V29.2D, V16.2D + mov V29.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V29.4S, V18.4S + mov V29.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V29.2D, V18.2D + mov V29.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V29.4S, V20.4S + mov V29.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V29.2D, V20.2D + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x1] + ldp Q7, Q8, [x1, #32] + ldp Q9, Q10, [x1, #64] + ldp Q11, Q12, [x1, #96] + ldp Q13, Q14, [x1, #128] + ldp Q15, Q16, [x1, #160] + ldp Q17, Q18, [x1, #192] + ldp Q19, Q20, [x1, #224] + ldr Q0, [x2, #48] + ldr Q1, [x3, #48] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #192] + ldr Q2, [x2, #208] + ldr Q1, [x3, #192] + ldr Q3, [x3, #208] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #224] + ldr Q2, [x2, #240] + ldr Q1, [x3, #224] + ldr Q3, [x3, #240] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #256] + ldr Q2, [x2, #272] + ldr Q1, [x3, #256] + ldr Q3, [x3, #272] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #288] + ldr Q2, [x2, #304] + ldr Q1, [x3, #288] + ldr Q3, [x3, #304] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #448] + ldr Q2, [x2, #464] + ldr Q1, [x3, #448] + ldr Q3, [x3, #464] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V21.8H, V21.8H, V29.8H + sub V22.8H, V22.8H, V30.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #480] + ldr Q2, [x2, #496] + ldr Q1, [x3, #480] + ldr Q3, [x3, #496] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V23.8H, V23.8H, V29.8H + sub V24.8H, V24.8H, V30.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #512] + ldr Q2, [x2, #528] + ldr Q1, [x3, #512] + ldr Q3, [x3, #528] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V25.8H, V25.8H, V29.8H + sub V26.8H, V26.8H, V30.8H + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #544] + ldr Q2, [x2, #560] + ldr Q1, [x3, #544] + ldr Q3, [x3, #560] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmulh V29.8H, V29.8H, V4.H[0] + sqrdmulh V30.8H, V30.8H, V4.H[0] + sub V27.8H, V27.8H, V29.8H + sub V28.8H, V28.8H, V30.8H + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + sqdmulh V21.8H, V5.8H, V4.H[2] + sqdmulh V22.8H, V6.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V5.8H, V21.8H, V4.H[0] + mls V6.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V7.8H, V4.H[2] + sqdmulh V22.8H, V8.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V7.8H, V21.8H, V4.H[0] + mls V8.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V9.8H, V4.H[2] + sqdmulh V22.8H, V10.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V9.8H, V21.8H, V4.H[0] + mls V10.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V11.8H, V4.H[2] + sqdmulh V22.8H, V12.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V11.8H, V21.8H, V4.H[0] + mls V12.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V13.8H, V4.H[2] + sqdmulh V22.8H, V14.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V13.8H, V21.8H, V4.H[0] + mls V14.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V15.8H, V4.H[2] + sqdmulh V22.8H, V16.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V15.8H, V21.8H, V4.H[0] + mls V16.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V17.8H, V4.H[2] + sqdmulh V22.8H, V18.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V17.8H, V21.8H, V4.H[0] + mls V18.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V19.8H, V4.H[2] + sqdmulh V22.8H, V20.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V19.8H, V21.8H, V4.H[0] + mls V20.8H, V22.8H, V4.H[0] + mov V29.16B, V5.16B + trn1 V5.4S, V5.4S, V6.4S + trn2 V6.4S, V29.4S, V6.4S + mov V29.16B, V5.16B + trn1 V5.2D, V5.2D, V6.2D + trn2 V6.2D, V29.2D, V6.2D + mov V29.16B, V7.16B + trn1 V7.4S, V7.4S, V8.4S + trn2 V8.4S, V29.4S, V8.4S + mov V29.16B, V7.16B + trn1 V7.2D, V7.2D, V8.2D + trn2 V8.2D, V29.2D, V8.2D + mov V29.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V29.4S, V10.4S + mov V29.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V29.2D, V10.2D + mov V29.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V29.4S, V12.4S + mov V29.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V29.2D, V12.2D + mov V29.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V29.4S, V14.4S + mov V29.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V29.2D, V14.2D + mov V29.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V29.4S, V16.4S + mov V29.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V29.2D, V16.2D + mov V29.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V29.4S, V18.4S + mov V29.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V29.2D, V18.2D + mov V29.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V29.4S, V20.4S + mov V29.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V29.2D, V20.2D + stp Q5, Q6, [x1] + stp Q7, Q8, [x1, #32] + stp Q9, Q10, [x1, #64] + stp Q11, Q12, [x1, #96] + stp Q13, Q14, [x1, #128] + stp Q15, Q16, [x1, #160] + stp Q17, Q18, [x1, #192] + stp Q19, Q20, [x1, #224] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_zetas_inv DCW 0x06a5, 0x06a5, 0x070f, 0x070f, 0x05b4, 0x05b4, 0x0943, 0x0943 @@ -1415,7 +1415,7 @@ L_mlkem_aarch64_zetas_inv DCW 0x0082, 0x0642, 0x074f, 0x033d, 0x0b82, 0x0bf9, 0x052d, 0x0ac4 DCW 0x0745, 0x05c2, 0x04b2, 0x093f, 0x0c4b, 0x06d8, 0x0a93, 0x00ab DCW 0x0c37, 0x0be2, 0x0773, 0x072c, 0x05ed, 0x0167, 0x02f6, 0x05a1 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_zetas_inv_qinv DCW 0xa5a5, 0xa5a5, 0x440f, 0x440f, 0xe1b4, 0xe1b4, 0xa243, 0xa243 @@ -1460,10 +1460,10 @@ L_mlkem_aarch64_zetas_inv_qinv mlkem_invntt PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_zetas_inv add x2, x2, L_mlkem_aarch64_zetas_inv adrp x3, L_mlkem_aarch64_zetas_inv_qinv @@ -1471,1411 +1471,1411 @@ mlkem_invntt PROC adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts add x1, x0, #0x100 - ldr q8, [x4] - ldp q9, q10, [x0] - ldp q11, q12, [x0, #32] - ldp q13, q14, [x0, #64] - ldp q15, q16, [x0, #96] - ldp q17, q18, [x0, #128] - ldp q19, q20, [x0, #160] - ldp q21, q22, [x0, #192] - ldp q23, q24, [x0, #224] - mov v25.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v25.2d, v10.2d - mov v25.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v25.4s, v10.4s - mov v25.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v25.2d, v12.2d - mov v25.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v25.4s, v12.4s - mov v25.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v25.2d, v14.2d - mov v25.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v25.4s, v14.4s - mov v25.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v25.2d, v16.2d - mov v25.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v25.4s, v16.4s - mov v25.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v25.2d, v18.2d - mov v25.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v25.4s, v18.4s - mov v25.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v25.2d, v20.2d - mov v25.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v25.4s, v20.4s - mov v25.16b, v21.16b - trn1 v21.2d, v21.2d, v22.2d - trn2 v22.2d, v25.2d, v22.2d - mov v25.16b, v21.16b - trn1 v21.4s, v21.4s, v22.4s - trn2 v22.4s, v25.4s, v22.4s - mov v25.16b, v23.16b - trn1 v23.2d, v23.2d, v24.2d - trn2 v24.2d, v25.2d, v24.2d - mov v25.16b, v23.16b - trn1 v23.4s, v23.4s, v24.4s - trn2 v24.4s, v25.4s, v24.4s - ldr q0, [x2] - ldr q1, [x2, #16] - ldr q2, [x3] - ldr q3, [x3, #16] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #32] - ldr q1, [x2, #48] - ldr q2, [x3, #32] - ldr q3, [x3, #48] - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #64] - ldr q1, [x2, #80] - ldr q2, [x3, #64] - ldr q3, [x3, #80] - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #96] - ldr q1, [x2, #112] - ldr q2, [x3, #96] - ldr q3, [x3, #112] - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #256] - ldr q1, [x2, #272] - ldr q2, [x3, #256] - ldr q3, [x3, #272] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v25.4s, v10.4s - trn2 v12.4s, v26.4s, v12.4s - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #288] - ldr q1, [x2, #304] - ldr q2, [x3, #288] - ldr q3, [x3, #304] - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v25.4s, v14.4s - trn2 v16.4s, v26.4s, v16.4s - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #320] - ldr q1, [x2, #336] - ldr q2, [x3, #320] - ldr q3, [x3, #336] - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v25.4s, v18.4s - trn2 v20.4s, v26.4s, v20.4s - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #352] - ldr q1, [x2, #368] - ldr q2, [x3, #352] - ldr q3, [x3, #368] - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.4s, v21.4s, v22.4s - trn1 v23.4s, v23.4s, v24.4s - trn2 v22.4s, v25.4s, v22.4s - trn2 v24.4s, v26.4s, v24.4s - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #512] - ldr q2, [x3, #512] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v25.2d, v10.2d - trn2 v12.2d, v26.2d, v12.2d - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.h[0] - mul v27.8h, v28.8h, v2.h[1] - sqrdmulh v10.8h, v26.8h, v0.h[0] - sqrdmulh v12.8h, v28.8h, v0.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v25.2d, v14.2d - trn2 v16.2d, v26.2d, v16.2d - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.h[2] - mul v27.8h, v28.8h, v2.h[3] - sqrdmulh v14.8h, v26.8h, v0.h[2] - sqrdmulh v16.8h, v28.8h, v0.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v25.2d, v18.2d - trn2 v20.2d, v26.2d, v20.2d - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.h[4] - mul v27.8h, v28.8h, v2.h[5] - sqrdmulh v18.8h, v26.8h, v0.h[4] - sqrdmulh v20.8h, v28.8h, v0.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.2d, v21.2d, v22.2d - trn1 v23.2d, v23.2d, v24.2d - trn2 v22.2d, v25.2d, v22.2d - trn2 v24.2d, v26.2d, v24.2d - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.h[6] - mul v27.8h, v28.8h, v2.h[7] - sqrdmulh v22.8h, v26.8h, v0.h[6] - sqrdmulh v24.8h, v28.8h, v0.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v11.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v11.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v13.8h, v8.h[2] - sqdmulh v26.8h, v15.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v13.8h, v25.8h, v8.h[0] - mls v15.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v19.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v19.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v21.8h, v8.h[2] - sqdmulh v26.8h, v23.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v21.8h, v25.8h, v8.h[0] - mls v23.8h, v26.8h, v8.h[0] - stp q9, q10, [x0] - stp q11, q12, [x0, #32] - stp q13, q14, [x0, #64] - stp q15, q16, [x0, #96] - stp q17, q18, [x0, #128] - stp q19, q20, [x0, #160] - stp q21, q22, [x0, #192] - stp q23, q24, [x0, #224] - ldp q9, q10, [x1] - ldp q11, q12, [x1, #32] - ldp q13, q14, [x1, #64] - ldp q15, q16, [x1, #96] - ldp q17, q18, [x1, #128] - ldp q19, q20, [x1, #160] - ldp q21, q22, [x1, #192] - ldp q23, q24, [x1, #224] - mov v25.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v25.2d, v10.2d - mov v25.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v25.4s, v10.4s - mov v25.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v25.2d, v12.2d - mov v25.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v25.4s, v12.4s - mov v25.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v25.2d, v14.2d - mov v25.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v25.4s, v14.4s - mov v25.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v25.2d, v16.2d - mov v25.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v25.4s, v16.4s - mov v25.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v25.2d, v18.2d - mov v25.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v25.4s, v18.4s - mov v25.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v25.2d, v20.2d - mov v25.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v25.4s, v20.4s - mov v25.16b, v21.16b - trn1 v21.2d, v21.2d, v22.2d - trn2 v22.2d, v25.2d, v22.2d - mov v25.16b, v21.16b - trn1 v21.4s, v21.4s, v22.4s - trn2 v22.4s, v25.4s, v22.4s - mov v25.16b, v23.16b - trn1 v23.2d, v23.2d, v24.2d - trn2 v24.2d, v25.2d, v24.2d - mov v25.16b, v23.16b - trn1 v23.4s, v23.4s, v24.4s - trn2 v24.4s, v25.4s, v24.4s - ldr q0, [x2, #128] - ldr q1, [x2, #144] - ldr q2, [x3, #128] - ldr q3, [x3, #144] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #160] - ldr q1, [x2, #176] - ldr q2, [x3, #160] - ldr q3, [x3, #176] - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #192] - ldr q1, [x2, #208] - ldr q2, [x3, #192] - ldr q3, [x3, #208] - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #224] - ldr q1, [x2, #240] - ldr q2, [x3, #224] - ldr q3, [x3, #240] - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #384] - ldr q1, [x2, #400] - ldr q2, [x3, #384] - ldr q3, [x3, #400] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v25.4s, v10.4s - trn2 v12.4s, v26.4s, v12.4s - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #416] - ldr q1, [x2, #432] - ldr q2, [x3, #416] - ldr q3, [x3, #432] - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v25.4s, v14.4s - trn2 v16.4s, v26.4s, v16.4s - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #448] - ldr q1, [x2, #464] - ldr q2, [x3, #448] - ldr q3, [x3, #464] - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v25.4s, v18.4s - trn2 v20.4s, v26.4s, v20.4s - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #480] - ldr q1, [x2, #496] - ldr q2, [x3, #480] - ldr q3, [x3, #496] - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.4s, v21.4s, v22.4s - trn1 v23.4s, v23.4s, v24.4s - trn2 v22.4s, v25.4s, v22.4s - trn2 v24.4s, v26.4s, v24.4s - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #528] - ldr q2, [x3, #528] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v25.2d, v10.2d - trn2 v12.2d, v26.2d, v12.2d - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.h[0] - mul v27.8h, v28.8h, v2.h[1] - sqrdmulh v10.8h, v26.8h, v0.h[0] - sqrdmulh v12.8h, v28.8h, v0.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v25.2d, v14.2d - trn2 v16.2d, v26.2d, v16.2d - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.h[2] - mul v27.8h, v28.8h, v2.h[3] - sqrdmulh v14.8h, v26.8h, v0.h[2] - sqrdmulh v16.8h, v28.8h, v0.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v25.2d, v18.2d - trn2 v20.2d, v26.2d, v20.2d - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.h[4] - mul v27.8h, v28.8h, v2.h[5] - sqrdmulh v18.8h, v26.8h, v0.h[4] - sqrdmulh v20.8h, v28.8h, v0.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.2d, v21.2d, v22.2d - trn1 v23.2d, v23.2d, v24.2d - trn2 v22.2d, v25.2d, v22.2d - trn2 v24.2d, v26.2d, v24.2d - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.h[6] - mul v27.8h, v28.8h, v2.h[7] - sqrdmulh v22.8h, v26.8h, v0.h[6] - sqrdmulh v24.8h, v28.8h, v0.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v11.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v11.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v13.8h, v8.h[2] - sqdmulh v26.8h, v15.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v13.8h, v25.8h, v8.h[0] - mls v15.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v19.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v19.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v21.8h, v8.h[2] - sqdmulh v26.8h, v23.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v21.8h, v25.8h, v8.h[0] - mls v23.8h, v26.8h, v8.h[0] - stp q9, q10, [x1] - stp q11, q12, [x1, #32] - stp q13, q14, [x1, #64] - stp q15, q16, [x1, #96] - stp q17, q18, [x1, #128] - stp q19, q20, [x1, #160] - stp q21, q22, [x1, #192] - stp q23, q24, [x1, #224] - ldr q4, [x2, #544] - ldr q5, [x2, #560] - ldr q6, [x3, #544] - ldr q7, [x3, #560] - ldr q9, [x0] - ldr q10, [x0, #32] - ldr q11, [x0, #64] - ldr q12, [x0, #96] - ldr q13, [x0, #128] - ldr q14, [x0, #160] - ldr q15, [x0, #192] - ldr q16, [x0, #224] - ldr q17, [x1] - ldr q18, [x1, #32] - ldr q19, [x1, #64] - ldr q20, [x1, #96] - ldr q21, [x1, #128] - ldr q22, [x1, #160] - ldr q23, [x1, #192] - ldr q24, [x1, #224] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v6.h[0] - mul v27.8h, v28.8h, v6.h[1] - sqrdmulh v10.8h, v26.8h, v4.h[0] - sqrdmulh v12.8h, v28.8h, v4.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v6.h[2] - mul v27.8h, v28.8h, v6.h[3] - sqrdmulh v14.8h, v26.8h, v4.h[2] - sqrdmulh v16.8h, v28.8h, v4.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v6.h[4] - mul v27.8h, v28.8h, v6.h[5] - sqrdmulh v18.8h, v26.8h, v4.h[4] - sqrdmulh v20.8h, v28.8h, v4.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v6.h[6] - mul v27.8h, v28.8h, v6.h[7] - sqrdmulh v22.8h, v26.8h, v4.h[6] - sqrdmulh v24.8h, v28.8h, v4.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v11.8h - sub v28.8h, v10.8h, v12.8h - add v9.8h, v9.8h, v11.8h - add v10.8h, v10.8h, v12.8h - mul v25.8h, v26.8h, v7.h[0] - mul v27.8h, v28.8h, v7.h[0] - sqrdmulh v11.8h, v26.8h, v5.h[0] - sqrdmulh v12.8h, v28.8h, v5.h[0] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v11.8h, v11.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v15.8h - sub v28.8h, v14.8h, v16.8h - add v13.8h, v13.8h, v15.8h - add v14.8h, v14.8h, v16.8h - mul v25.8h, v26.8h, v7.h[1] - mul v27.8h, v28.8h, v7.h[1] - sqrdmulh v15.8h, v26.8h, v5.h[1] - sqrdmulh v16.8h, v28.8h, v5.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v19.8h - sub v28.8h, v18.8h, v20.8h - add v17.8h, v17.8h, v19.8h - add v18.8h, v18.8h, v20.8h - mul v25.8h, v26.8h, v7.h[2] - mul v27.8h, v28.8h, v7.h[2] - sqrdmulh v19.8h, v26.8h, v5.h[2] - sqrdmulh v20.8h, v28.8h, v5.h[2] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v23.8h - sub v28.8h, v22.8h, v24.8h - add v21.8h, v21.8h, v23.8h - add v22.8h, v22.8h, v24.8h - mul v25.8h, v26.8h, v7.h[3] - mul v27.8h, v28.8h, v7.h[3] - sqrdmulh v23.8h, v26.8h, v5.h[3] - sqrdmulh v24.8h, v28.8h, v5.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v13.8h - sub v28.8h, v10.8h, v14.8h - add v9.8h, v9.8h, v13.8h - add v10.8h, v10.8h, v14.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v13.8h, v26.8h, v5.h[4] - sqrdmulh v14.8h, v28.8h, v5.h[4] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v13.8h, v13.8h, v25.8h - sub v14.8h, v14.8h, v27.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - sub v26.8h, v11.8h, v15.8h - sub v28.8h, v12.8h, v16.8h - add v11.8h, v11.8h, v15.8h - add v12.8h, v12.8h, v16.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v15.8h, v26.8h, v5.h[4] - sqrdmulh v16.8h, v28.8h, v5.h[4] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v21.8h - sub v28.8h, v18.8h, v22.8h - add v17.8h, v17.8h, v21.8h - add v18.8h, v18.8h, v22.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v21.8h, v26.8h, v5.h[5] - sqrdmulh v22.8h, v28.8h, v5.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v27.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v19.8h, v23.8h - sub v28.8h, v20.8h, v24.8h - add v19.8h, v19.8h, v23.8h - add v20.8h, v20.8h, v24.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v23.8h, v26.8h, v5.h[5] - sqrdmulh v24.8h, v28.8h, v5.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v10.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v10.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v11.8h, v8.h[2] - sqdmulh v26.8h, v12.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v11.8h, v25.8h, v8.h[0] - mls v12.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v18.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v18.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v19.8h, v8.h[2] - sqdmulh v26.8h, v20.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v19.8h, v25.8h, v8.h[0] - mls v20.8h, v26.8h, v8.h[0] - sub v26.8h, v9.8h, v17.8h - sub v28.8h, v10.8h, v18.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v17.8h, v26.8h, v5.h[6] - sqrdmulh v18.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v17.8h, v17.8h, v25.8h - sub v18.8h, v18.8h, v27.8h - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - sub v26.8h, v11.8h, v19.8h - sub v28.8h, v12.8h, v20.8h - add v11.8h, v11.8h, v19.8h - add v12.8h, v12.8h, v20.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v19.8h, v26.8h, v5.h[6] - sqrdmulh v20.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v13.8h, v21.8h - sub v28.8h, v14.8h, v22.8h - add v13.8h, v13.8h, v21.8h - add v14.8h, v14.8h, v22.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v21.8h, v26.8h, v5.h[6] - sqrdmulh v22.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v27.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v15.8h, v23.8h - sub v28.8h, v16.8h, v24.8h - add v15.8h, v15.8h, v23.8h - add v16.8h, v16.8h, v24.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v23.8h, v26.8h, v5.h[6] - sqrdmulh v24.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v25.8h, v9.8h, v7.h[7] - mul v26.8h, v10.8h, v7.h[7] - sqrdmulh v9.8h, v9.8h, v5.h[7] - sqrdmulh v10.8h, v10.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v9.8h, v9.8h, v25.8h - sub v10.8h, v10.8h, v26.8h - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v25.8h, v11.8h, v7.h[7] - mul v26.8h, v12.8h, v7.h[7] - sqrdmulh v11.8h, v11.8h, v5.h[7] - sqrdmulh v12.8h, v12.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v11.8h, v11.8h, v25.8h - sub v12.8h, v12.8h, v26.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v25.8h, v13.8h, v7.h[7] - mul v26.8h, v14.8h, v7.h[7] - sqrdmulh v13.8h, v13.8h, v5.h[7] - sqrdmulh v14.8h, v14.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v13.8h, v13.8h, v25.8h - sub v14.8h, v14.8h, v26.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v25.8h, v15.8h, v7.h[7] - mul v26.8h, v16.8h, v7.h[7] - sqrdmulh v15.8h, v15.8h, v5.h[7] - sqrdmulh v16.8h, v16.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v26.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - mul v25.8h, v17.8h, v7.h[7] - mul v26.8h, v18.8h, v7.h[7] - sqrdmulh v17.8h, v17.8h, v5.h[7] - sqrdmulh v18.8h, v18.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v17.8h, v17.8h, v25.8h - sub v18.8h, v18.8h, v26.8h - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - mul v25.8h, v19.8h, v7.h[7] - mul v26.8h, v20.8h, v7.h[7] - sqrdmulh v19.8h, v19.8h, v5.h[7] - sqrdmulh v20.8h, v20.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v26.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - mul v25.8h, v21.8h, v7.h[7] - mul v26.8h, v22.8h, v7.h[7] - sqrdmulh v21.8h, v21.8h, v5.h[7] - sqrdmulh v22.8h, v22.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v26.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v25.8h, v23.8h, v7.h[7] - mul v26.8h, v24.8h, v7.h[7] - sqrdmulh v23.8h, v23.8h, v5.h[7] - sqrdmulh v24.8h, v24.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v26.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - str q9, [x0] - str q10, [x0, #32] - str q11, [x0, #64] - str q12, [x0, #96] - str q13, [x0, #128] - str q14, [x0, #160] - str q15, [x0, #192] - str q16, [x0, #224] - str q17, [x1] - str q18, [x1, #32] - str q19, [x1, #64] - str q20, [x1, #96] - str q21, [x1, #128] - str q22, [x1, #160] - str q23, [x1, #192] - str q24, [x1, #224] - ldr q9, [x0, #16] - ldr q10, [x0, #48] - ldr q11, [x0, #80] - ldr q12, [x0, #112] - ldr q13, [x0, #144] - ldr q14, [x0, #176] - ldr q15, [x0, #208] - ldr q16, [x0, #240] - ldr q17, [x1, #16] - ldr q18, [x1, #48] - ldr q19, [x1, #80] - ldr q20, [x1, #112] - ldr q21, [x1, #144] - ldr q22, [x1, #176] - ldr q23, [x1, #208] - ldr q24, [x1, #240] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v6.h[0] - mul v27.8h, v28.8h, v6.h[1] - sqrdmulh v10.8h, v26.8h, v4.h[0] - sqrdmulh v12.8h, v28.8h, v4.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v10.8h, v10.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v6.h[2] - mul v27.8h, v28.8h, v6.h[3] - sqrdmulh v14.8h, v26.8h, v4.h[2] - sqrdmulh v16.8h, v28.8h, v4.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v14.8h, v14.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v6.h[4] - mul v27.8h, v28.8h, v6.h[5] - sqrdmulh v18.8h, v26.8h, v4.h[4] - sqrdmulh v20.8h, v28.8h, v4.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v18.8h, v18.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v6.h[6] - mul v27.8h, v28.8h, v6.h[7] - sqrdmulh v22.8h, v26.8h, v4.h[6] - sqrdmulh v24.8h, v28.8h, v4.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v22.8h, v22.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v11.8h - sub v28.8h, v10.8h, v12.8h - add v9.8h, v9.8h, v11.8h - add v10.8h, v10.8h, v12.8h - mul v25.8h, v26.8h, v7.h[0] - mul v27.8h, v28.8h, v7.h[0] - sqrdmulh v11.8h, v26.8h, v5.h[0] - sqrdmulh v12.8h, v28.8h, v5.h[0] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v11.8h, v11.8h, v25.8h - sub v12.8h, v12.8h, v27.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v15.8h - sub v28.8h, v14.8h, v16.8h - add v13.8h, v13.8h, v15.8h - add v14.8h, v14.8h, v16.8h - mul v25.8h, v26.8h, v7.h[1] - mul v27.8h, v28.8h, v7.h[1] - sqrdmulh v15.8h, v26.8h, v5.h[1] - sqrdmulh v16.8h, v28.8h, v5.h[1] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v19.8h - sub v28.8h, v18.8h, v20.8h - add v17.8h, v17.8h, v19.8h - add v18.8h, v18.8h, v20.8h - mul v25.8h, v26.8h, v7.h[2] - mul v27.8h, v28.8h, v7.h[2] - sqrdmulh v19.8h, v26.8h, v5.h[2] - sqrdmulh v20.8h, v28.8h, v5.h[2] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v23.8h - sub v28.8h, v22.8h, v24.8h - add v21.8h, v21.8h, v23.8h - add v22.8h, v22.8h, v24.8h - mul v25.8h, v26.8h, v7.h[3] - mul v27.8h, v28.8h, v7.h[3] - sqrdmulh v23.8h, v26.8h, v5.h[3] - sqrdmulh v24.8h, v28.8h, v5.h[3] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v13.8h - sub v28.8h, v10.8h, v14.8h - add v9.8h, v9.8h, v13.8h - add v10.8h, v10.8h, v14.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v13.8h, v26.8h, v5.h[4] - sqrdmulh v14.8h, v28.8h, v5.h[4] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v13.8h, v13.8h, v25.8h - sub v14.8h, v14.8h, v27.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - sub v26.8h, v11.8h, v15.8h - sub v28.8h, v12.8h, v16.8h - add v11.8h, v11.8h, v15.8h - add v12.8h, v12.8h, v16.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v15.8h, v26.8h, v5.h[4] - sqrdmulh v16.8h, v28.8h, v5.h[4] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v27.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v21.8h - sub v28.8h, v18.8h, v22.8h - add v17.8h, v17.8h, v21.8h - add v18.8h, v18.8h, v22.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v21.8h, v26.8h, v5.h[5] - sqrdmulh v22.8h, v28.8h, v5.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v27.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v19.8h, v23.8h - sub v28.8h, v20.8h, v24.8h - add v19.8h, v19.8h, v23.8h - add v20.8h, v20.8h, v24.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v23.8h, v26.8h, v5.h[5] - sqrdmulh v24.8h, v28.8h, v5.h[5] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v10.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v10.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v11.8h, v8.h[2] - sqdmulh v26.8h, v12.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v11.8h, v25.8h, v8.h[0] - mls v12.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v18.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v18.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v19.8h, v8.h[2] - sqdmulh v26.8h, v20.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v19.8h, v25.8h, v8.h[0] - mls v20.8h, v26.8h, v8.h[0] - sub v26.8h, v9.8h, v17.8h - sub v28.8h, v10.8h, v18.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v17.8h, v26.8h, v5.h[6] - sqrdmulh v18.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v17.8h, v17.8h, v25.8h - sub v18.8h, v18.8h, v27.8h - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - sub v26.8h, v11.8h, v19.8h - sub v28.8h, v12.8h, v20.8h - add v11.8h, v11.8h, v19.8h - add v12.8h, v12.8h, v20.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v19.8h, v26.8h, v5.h[6] - sqrdmulh v20.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v27.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v13.8h, v21.8h - sub v28.8h, v14.8h, v22.8h - add v13.8h, v13.8h, v21.8h - add v14.8h, v14.8h, v22.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v21.8h, v26.8h, v5.h[6] - sqrdmulh v22.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v27.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v15.8h, v23.8h - sub v28.8h, v16.8h, v24.8h - add v15.8h, v15.8h, v23.8h - add v16.8h, v16.8h, v24.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v23.8h, v26.8h, v5.h[6] - sqrdmulh v24.8h, v28.8h, v5.h[6] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v27.8h, v27.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v27.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v25.8h, v9.8h, v7.h[7] - mul v26.8h, v10.8h, v7.h[7] - sqrdmulh v9.8h, v9.8h, v5.h[7] - sqrdmulh v10.8h, v10.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v9.8h, v9.8h, v25.8h - sub v10.8h, v10.8h, v26.8h - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v25.8h, v11.8h, v7.h[7] - mul v26.8h, v12.8h, v7.h[7] - sqrdmulh v11.8h, v11.8h, v5.h[7] - sqrdmulh v12.8h, v12.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v11.8h, v11.8h, v25.8h - sub v12.8h, v12.8h, v26.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v25.8h, v13.8h, v7.h[7] - mul v26.8h, v14.8h, v7.h[7] - sqrdmulh v13.8h, v13.8h, v5.h[7] - sqrdmulh v14.8h, v14.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v13.8h, v13.8h, v25.8h - sub v14.8h, v14.8h, v26.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v25.8h, v15.8h, v7.h[7] - mul v26.8h, v16.8h, v7.h[7] - sqrdmulh v15.8h, v15.8h, v5.h[7] - sqrdmulh v16.8h, v16.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v15.8h, v15.8h, v25.8h - sub v16.8h, v16.8h, v26.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - mul v25.8h, v17.8h, v7.h[7] - mul v26.8h, v18.8h, v7.h[7] - sqrdmulh v17.8h, v17.8h, v5.h[7] - sqrdmulh v18.8h, v18.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v17.8h, v17.8h, v25.8h - sub v18.8h, v18.8h, v26.8h - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - mul v25.8h, v19.8h, v7.h[7] - mul v26.8h, v20.8h, v7.h[7] - sqrdmulh v19.8h, v19.8h, v5.h[7] - sqrdmulh v20.8h, v20.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v19.8h, v19.8h, v25.8h - sub v20.8h, v20.8h, v26.8h - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - mul v25.8h, v21.8h, v7.h[7] - mul v26.8h, v22.8h, v7.h[7] - sqrdmulh v21.8h, v21.8h, v5.h[7] - sqrdmulh v22.8h, v22.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v21.8h, v21.8h, v25.8h - sub v22.8h, v22.8h, v26.8h - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v25.8h, v23.8h, v7.h[7] - mul v26.8h, v24.8h, v7.h[7] - sqrdmulh v23.8h, v23.8h, v5.h[7] - sqrdmulh v24.8h, v24.8h, v5.h[7] - sqrdmulh v25.8h, v25.8h, v8.h[0] - sqrdmulh v26.8h, v26.8h, v8.h[0] - sub v23.8h, v23.8h, v25.8h - sub v24.8h, v24.8h, v26.8h - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - str q9, [x0, #16] - str q10, [x0, #48] - str q11, [x0, #80] - str q12, [x0, #112] - str q13, [x0, #144] - str q14, [x0, #176] - str q15, [x0, #208] - str q16, [x0, #240] - str q17, [x1, #16] - str q18, [x1, #48] - str q19, [x1, #80] - str q20, [x1, #112] - str q21, [x1, #144] - str q22, [x1, #176] - str q23, [x1, #208] - str q24, [x1, #240] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q8, [x4] + ldp Q9, Q10, [x0] + ldp Q11, Q12, [x0, #32] + ldp Q13, Q14, [x0, #64] + ldp Q15, Q16, [x0, #96] + ldp Q17, Q18, [x0, #128] + ldp Q19, Q20, [x0, #160] + ldp Q21, Q22, [x0, #192] + ldp Q23, Q24, [x0, #224] + mov V25.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V25.2D, V10.2D + mov V25.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V25.4S, V10.4S + mov V25.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V25.2D, V12.2D + mov V25.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V25.4S, V12.4S + mov V25.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V25.2D, V14.2D + mov V25.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V25.4S, V14.4S + mov V25.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V25.2D, V16.2D + mov V25.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V25.4S, V16.4S + mov V25.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V25.2D, V18.2D + mov V25.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V25.4S, V18.4S + mov V25.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V25.2D, V20.2D + mov V25.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V25.4S, V20.4S + mov V25.16B, V21.16B + trn1 V21.2D, V21.2D, V22.2D + trn2 V22.2D, V25.2D, V22.2D + mov V25.16B, V21.16B + trn1 V21.4S, V21.4S, V22.4S + trn2 V22.4S, V25.4S, V22.4S + mov V25.16B, V23.16B + trn1 V23.2D, V23.2D, V24.2D + trn2 V24.2D, V25.2D, V24.2D + mov V25.16B, V23.16B + trn1 V23.4S, V23.4S, V24.4S + trn2 V24.4S, V25.4S, V24.4S + ldr Q0, [x2] + ldr Q1, [x2, #16] + ldr Q2, [x3] + ldr Q3, [x3, #16] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #32] + ldr Q1, [x2, #48] + ldr Q2, [x3, #32] + ldr Q3, [x3, #48] + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #64] + ldr Q1, [x2, #80] + ldr Q2, [x3, #64] + ldr Q3, [x3, #80] + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #96] + ldr Q1, [x2, #112] + ldr Q2, [x3, #96] + ldr Q3, [x3, #112] + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #256] + ldr Q1, [x2, #272] + ldr Q2, [x3, #256] + ldr Q3, [x3, #272] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V25.4S, V10.4S + trn2 V12.4S, V26.4S, V12.4S + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #288] + ldr Q1, [x2, #304] + ldr Q2, [x3, #288] + ldr Q3, [x3, #304] + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V25.4S, V14.4S + trn2 V16.4S, V26.4S, V16.4S + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #320] + ldr Q1, [x2, #336] + ldr Q2, [x3, #320] + ldr Q3, [x3, #336] + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V25.4S, V18.4S + trn2 V20.4S, V26.4S, V20.4S + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #352] + ldr Q1, [x2, #368] + ldr Q2, [x3, #352] + ldr Q3, [x3, #368] + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.4S, V21.4S, V22.4S + trn1 V23.4S, V23.4S, V24.4S + trn2 V22.4S, V25.4S, V22.4S + trn2 V24.4S, V26.4S, V24.4S + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #512] + ldr Q2, [x3, #512] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V25.2D, V10.2D + trn2 V12.2D, V26.2D, V12.2D + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.H[0] + mul V27.8H, V28.8H, V2.H[1] + sqrdmulh V10.8H, V26.8H, V0.H[0] + sqrdmulh V12.8H, V28.8H, V0.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V25.2D, V14.2D + trn2 V16.2D, V26.2D, V16.2D + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.H[2] + mul V27.8H, V28.8H, V2.H[3] + sqrdmulh V14.8H, V26.8H, V0.H[2] + sqrdmulh V16.8H, V28.8H, V0.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V25.2D, V18.2D + trn2 V20.2D, V26.2D, V20.2D + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.H[4] + mul V27.8H, V28.8H, V2.H[5] + sqrdmulh V18.8H, V26.8H, V0.H[4] + sqrdmulh V20.8H, V28.8H, V0.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.2D, V21.2D, V22.2D + trn1 V23.2D, V23.2D, V24.2D + trn2 V22.2D, V25.2D, V22.2D + trn2 V24.2D, V26.2D, V24.2D + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.H[6] + mul V27.8H, V28.8H, V2.H[7] + sqrdmulh V22.8H, V26.8H, V0.H[6] + sqrdmulh V24.8H, V28.8H, V0.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V11.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V11.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V13.8H, V8.H[2] + sqdmulh V26.8H, V15.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V13.8H, V25.8H, V8.H[0] + mls V15.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V19.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V19.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V21.8H, V8.H[2] + sqdmulh V26.8H, V23.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V21.8H, V25.8H, V8.H[0] + mls V23.8H, V26.8H, V8.H[0] + stp Q9, Q10, [x0] + stp Q11, Q12, [x0, #32] + stp Q13, Q14, [x0, #64] + stp Q15, Q16, [x0, #96] + stp Q17, Q18, [x0, #128] + stp Q19, Q20, [x0, #160] + stp Q21, Q22, [x0, #192] + stp Q23, Q24, [x0, #224] + ldp Q9, Q10, [x1] + ldp Q11, Q12, [x1, #32] + ldp Q13, Q14, [x1, #64] + ldp Q15, Q16, [x1, #96] + ldp Q17, Q18, [x1, #128] + ldp Q19, Q20, [x1, #160] + ldp Q21, Q22, [x1, #192] + ldp Q23, Q24, [x1, #224] + mov V25.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V25.2D, V10.2D + mov V25.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V25.4S, V10.4S + mov V25.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V25.2D, V12.2D + mov V25.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V25.4S, V12.4S + mov V25.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V25.2D, V14.2D + mov V25.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V25.4S, V14.4S + mov V25.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V25.2D, V16.2D + mov V25.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V25.4S, V16.4S + mov V25.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V25.2D, V18.2D + mov V25.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V25.4S, V18.4S + mov V25.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V25.2D, V20.2D + mov V25.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V25.4S, V20.4S + mov V25.16B, V21.16B + trn1 V21.2D, V21.2D, V22.2D + trn2 V22.2D, V25.2D, V22.2D + mov V25.16B, V21.16B + trn1 V21.4S, V21.4S, V22.4S + trn2 V22.4S, V25.4S, V22.4S + mov V25.16B, V23.16B + trn1 V23.2D, V23.2D, V24.2D + trn2 V24.2D, V25.2D, V24.2D + mov V25.16B, V23.16B + trn1 V23.4S, V23.4S, V24.4S + trn2 V24.4S, V25.4S, V24.4S + ldr Q0, [x2, #128] + ldr Q1, [x2, #144] + ldr Q2, [x3, #128] + ldr Q3, [x3, #144] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #160] + ldr Q1, [x2, #176] + ldr Q2, [x3, #160] + ldr Q3, [x3, #176] + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #192] + ldr Q1, [x2, #208] + ldr Q2, [x3, #192] + ldr Q3, [x3, #208] + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #224] + ldr Q1, [x2, #240] + ldr Q2, [x3, #224] + ldr Q3, [x3, #240] + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #384] + ldr Q1, [x2, #400] + ldr Q2, [x3, #384] + ldr Q3, [x3, #400] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V25.4S, V10.4S + trn2 V12.4S, V26.4S, V12.4S + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #416] + ldr Q1, [x2, #432] + ldr Q2, [x3, #416] + ldr Q3, [x3, #432] + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V25.4S, V14.4S + trn2 V16.4S, V26.4S, V16.4S + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #448] + ldr Q1, [x2, #464] + ldr Q2, [x3, #448] + ldr Q3, [x3, #464] + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V25.4S, V18.4S + trn2 V20.4S, V26.4S, V20.4S + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #480] + ldr Q1, [x2, #496] + ldr Q2, [x3, #480] + ldr Q3, [x3, #496] + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.4S, V21.4S, V22.4S + trn1 V23.4S, V23.4S, V24.4S + trn2 V22.4S, V25.4S, V22.4S + trn2 V24.4S, V26.4S, V24.4S + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #528] + ldr Q2, [x3, #528] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V25.2D, V10.2D + trn2 V12.2D, V26.2D, V12.2D + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.H[0] + mul V27.8H, V28.8H, V2.H[1] + sqrdmulh V10.8H, V26.8H, V0.H[0] + sqrdmulh V12.8H, V28.8H, V0.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V25.2D, V14.2D + trn2 V16.2D, V26.2D, V16.2D + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.H[2] + mul V27.8H, V28.8H, V2.H[3] + sqrdmulh V14.8H, V26.8H, V0.H[2] + sqrdmulh V16.8H, V28.8H, V0.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V25.2D, V18.2D + trn2 V20.2D, V26.2D, V20.2D + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.H[4] + mul V27.8H, V28.8H, V2.H[5] + sqrdmulh V18.8H, V26.8H, V0.H[4] + sqrdmulh V20.8H, V28.8H, V0.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.2D, V21.2D, V22.2D + trn1 V23.2D, V23.2D, V24.2D + trn2 V22.2D, V25.2D, V22.2D + trn2 V24.2D, V26.2D, V24.2D + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.H[6] + mul V27.8H, V28.8H, V2.H[7] + sqrdmulh V22.8H, V26.8H, V0.H[6] + sqrdmulh V24.8H, V28.8H, V0.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V11.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V11.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V13.8H, V8.H[2] + sqdmulh V26.8H, V15.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V13.8H, V25.8H, V8.H[0] + mls V15.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V19.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V19.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V21.8H, V8.H[2] + sqdmulh V26.8H, V23.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V21.8H, V25.8H, V8.H[0] + mls V23.8H, V26.8H, V8.H[0] + stp Q9, Q10, [x1] + stp Q11, Q12, [x1, #32] + stp Q13, Q14, [x1, #64] + stp Q15, Q16, [x1, #96] + stp Q17, Q18, [x1, #128] + stp Q19, Q20, [x1, #160] + stp Q21, Q22, [x1, #192] + stp Q23, Q24, [x1, #224] + ldr Q4, [x2, #544] + ldr Q5, [x2, #560] + ldr Q6, [x3, #544] + ldr Q7, [x3, #560] + ldr Q9, [x0] + ldr Q10, [x0, #32] + ldr Q11, [x0, #64] + ldr Q12, [x0, #96] + ldr Q13, [x0, #128] + ldr Q14, [x0, #160] + ldr Q15, [x0, #192] + ldr Q16, [x0, #224] + ldr Q17, [x1] + ldr Q18, [x1, #32] + ldr Q19, [x1, #64] + ldr Q20, [x1, #96] + ldr Q21, [x1, #128] + ldr Q22, [x1, #160] + ldr Q23, [x1, #192] + ldr Q24, [x1, #224] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V6.H[0] + mul V27.8H, V28.8H, V6.H[1] + sqrdmulh V10.8H, V26.8H, V4.H[0] + sqrdmulh V12.8H, V28.8H, V4.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V6.H[2] + mul V27.8H, V28.8H, V6.H[3] + sqrdmulh V14.8H, V26.8H, V4.H[2] + sqrdmulh V16.8H, V28.8H, V4.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V6.H[4] + mul V27.8H, V28.8H, V6.H[5] + sqrdmulh V18.8H, V26.8H, V4.H[4] + sqrdmulh V20.8H, V28.8H, V4.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V6.H[6] + mul V27.8H, V28.8H, V6.H[7] + sqrdmulh V22.8H, V26.8H, V4.H[6] + sqrdmulh V24.8H, V28.8H, V4.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V11.8H + sub V28.8H, V10.8H, V12.8H + add V9.8H, V9.8H, V11.8H + add V10.8H, V10.8H, V12.8H + mul V25.8H, V26.8H, V7.H[0] + mul V27.8H, V28.8H, V7.H[0] + sqrdmulh V11.8H, V26.8H, V5.H[0] + sqrdmulh V12.8H, V28.8H, V5.H[0] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V11.8H, V11.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V15.8H + sub V28.8H, V14.8H, V16.8H + add V13.8H, V13.8H, V15.8H + add V14.8H, V14.8H, V16.8H + mul V25.8H, V26.8H, V7.H[1] + mul V27.8H, V28.8H, V7.H[1] + sqrdmulh V15.8H, V26.8H, V5.H[1] + sqrdmulh V16.8H, V28.8H, V5.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V19.8H + sub V28.8H, V18.8H, V20.8H + add V17.8H, V17.8H, V19.8H + add V18.8H, V18.8H, V20.8H + mul V25.8H, V26.8H, V7.H[2] + mul V27.8H, V28.8H, V7.H[2] + sqrdmulh V19.8H, V26.8H, V5.H[2] + sqrdmulh V20.8H, V28.8H, V5.H[2] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V23.8H + sub V28.8H, V22.8H, V24.8H + add V21.8H, V21.8H, V23.8H + add V22.8H, V22.8H, V24.8H + mul V25.8H, V26.8H, V7.H[3] + mul V27.8H, V28.8H, V7.H[3] + sqrdmulh V23.8H, V26.8H, V5.H[3] + sqrdmulh V24.8H, V28.8H, V5.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V13.8H + sub V28.8H, V10.8H, V14.8H + add V9.8H, V9.8H, V13.8H + add V10.8H, V10.8H, V14.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V13.8H, V26.8H, V5.H[4] + sqrdmulh V14.8H, V28.8H, V5.H[4] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V13.8H, V13.8H, V25.8H + sub V14.8H, V14.8H, V27.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + sub V26.8H, V11.8H, V15.8H + sub V28.8H, V12.8H, V16.8H + add V11.8H, V11.8H, V15.8H + add V12.8H, V12.8H, V16.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V15.8H, V26.8H, V5.H[4] + sqrdmulh V16.8H, V28.8H, V5.H[4] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V21.8H + sub V28.8H, V18.8H, V22.8H + add V17.8H, V17.8H, V21.8H + add V18.8H, V18.8H, V22.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V21.8H, V26.8H, V5.H[5] + sqrdmulh V22.8H, V28.8H, V5.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V27.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V19.8H, V23.8H + sub V28.8H, V20.8H, V24.8H + add V19.8H, V19.8H, V23.8H + add V20.8H, V20.8H, V24.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V23.8H, V26.8H, V5.H[5] + sqrdmulh V24.8H, V28.8H, V5.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V10.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V10.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V11.8H, V8.H[2] + sqdmulh V26.8H, V12.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V11.8H, V25.8H, V8.H[0] + mls V12.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V18.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V18.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V19.8H, V8.H[2] + sqdmulh V26.8H, V20.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V19.8H, V25.8H, V8.H[0] + mls V20.8H, V26.8H, V8.H[0] + sub V26.8H, V9.8H, V17.8H + sub V28.8H, V10.8H, V18.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V17.8H, V26.8H, V5.H[6] + sqrdmulh V18.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V17.8H, V17.8H, V25.8H + sub V18.8H, V18.8H, V27.8H + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + sub V26.8H, V11.8H, V19.8H + sub V28.8H, V12.8H, V20.8H + add V11.8H, V11.8H, V19.8H + add V12.8H, V12.8H, V20.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V19.8H, V26.8H, V5.H[6] + sqrdmulh V20.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V13.8H, V21.8H + sub V28.8H, V14.8H, V22.8H + add V13.8H, V13.8H, V21.8H + add V14.8H, V14.8H, V22.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V21.8H, V26.8H, V5.H[6] + sqrdmulh V22.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V27.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V15.8H, V23.8H + sub V28.8H, V16.8H, V24.8H + add V15.8H, V15.8H, V23.8H + add V16.8H, V16.8H, V24.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V23.8H, V26.8H, V5.H[6] + sqrdmulh V24.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V25.8H, V9.8H, V7.H[7] + mul V26.8H, V10.8H, V7.H[7] + sqrdmulh V9.8H, V9.8H, V5.H[7] + sqrdmulh V10.8H, V10.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V9.8H, V9.8H, V25.8H + sub V10.8H, V10.8H, V26.8H + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V25.8H, V11.8H, V7.H[7] + mul V26.8H, V12.8H, V7.H[7] + sqrdmulh V11.8H, V11.8H, V5.H[7] + sqrdmulh V12.8H, V12.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V11.8H, V11.8H, V25.8H + sub V12.8H, V12.8H, V26.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V25.8H, V13.8H, V7.H[7] + mul V26.8H, V14.8H, V7.H[7] + sqrdmulh V13.8H, V13.8H, V5.H[7] + sqrdmulh V14.8H, V14.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V13.8H, V13.8H, V25.8H + sub V14.8H, V14.8H, V26.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V25.8H, V15.8H, V7.H[7] + mul V26.8H, V16.8H, V7.H[7] + sqrdmulh V15.8H, V15.8H, V5.H[7] + sqrdmulh V16.8H, V16.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V26.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + mul V25.8H, V17.8H, V7.H[7] + mul V26.8H, V18.8H, V7.H[7] + sqrdmulh V17.8H, V17.8H, V5.H[7] + sqrdmulh V18.8H, V18.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V17.8H, V17.8H, V25.8H + sub V18.8H, V18.8H, V26.8H + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + mul V25.8H, V19.8H, V7.H[7] + mul V26.8H, V20.8H, V7.H[7] + sqrdmulh V19.8H, V19.8H, V5.H[7] + sqrdmulh V20.8H, V20.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V26.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + mul V25.8H, V21.8H, V7.H[7] + mul V26.8H, V22.8H, V7.H[7] + sqrdmulh V21.8H, V21.8H, V5.H[7] + sqrdmulh V22.8H, V22.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V26.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V25.8H, V23.8H, V7.H[7] + mul V26.8H, V24.8H, V7.H[7] + sqrdmulh V23.8H, V23.8H, V5.H[7] + sqrdmulh V24.8H, V24.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V26.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + str Q9, [x0] + str Q10, [x0, #32] + str Q11, [x0, #64] + str Q12, [x0, #96] + str Q13, [x0, #128] + str Q14, [x0, #160] + str Q15, [x0, #192] + str Q16, [x0, #224] + str Q17, [x1] + str Q18, [x1, #32] + str Q19, [x1, #64] + str Q20, [x1, #96] + str Q21, [x1, #128] + str Q22, [x1, #160] + str Q23, [x1, #192] + str Q24, [x1, #224] + ldr Q9, [x0, #16] + ldr Q10, [x0, #48] + ldr Q11, [x0, #80] + ldr Q12, [x0, #112] + ldr Q13, [x0, #144] + ldr Q14, [x0, #176] + ldr Q15, [x0, #208] + ldr Q16, [x0, #240] + ldr Q17, [x1, #16] + ldr Q18, [x1, #48] + ldr Q19, [x1, #80] + ldr Q20, [x1, #112] + ldr Q21, [x1, #144] + ldr Q22, [x1, #176] + ldr Q23, [x1, #208] + ldr Q24, [x1, #240] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V6.H[0] + mul V27.8H, V28.8H, V6.H[1] + sqrdmulh V10.8H, V26.8H, V4.H[0] + sqrdmulh V12.8H, V28.8H, V4.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V10.8H, V10.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V6.H[2] + mul V27.8H, V28.8H, V6.H[3] + sqrdmulh V14.8H, V26.8H, V4.H[2] + sqrdmulh V16.8H, V28.8H, V4.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V14.8H, V14.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V6.H[4] + mul V27.8H, V28.8H, V6.H[5] + sqrdmulh V18.8H, V26.8H, V4.H[4] + sqrdmulh V20.8H, V28.8H, V4.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V18.8H, V18.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V6.H[6] + mul V27.8H, V28.8H, V6.H[7] + sqrdmulh V22.8H, V26.8H, V4.H[6] + sqrdmulh V24.8H, V28.8H, V4.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V22.8H, V22.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V11.8H + sub V28.8H, V10.8H, V12.8H + add V9.8H, V9.8H, V11.8H + add V10.8H, V10.8H, V12.8H + mul V25.8H, V26.8H, V7.H[0] + mul V27.8H, V28.8H, V7.H[0] + sqrdmulh V11.8H, V26.8H, V5.H[0] + sqrdmulh V12.8H, V28.8H, V5.H[0] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V11.8H, V11.8H, V25.8H + sub V12.8H, V12.8H, V27.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V15.8H + sub V28.8H, V14.8H, V16.8H + add V13.8H, V13.8H, V15.8H + add V14.8H, V14.8H, V16.8H + mul V25.8H, V26.8H, V7.H[1] + mul V27.8H, V28.8H, V7.H[1] + sqrdmulh V15.8H, V26.8H, V5.H[1] + sqrdmulh V16.8H, V28.8H, V5.H[1] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V19.8H + sub V28.8H, V18.8H, V20.8H + add V17.8H, V17.8H, V19.8H + add V18.8H, V18.8H, V20.8H + mul V25.8H, V26.8H, V7.H[2] + mul V27.8H, V28.8H, V7.H[2] + sqrdmulh V19.8H, V26.8H, V5.H[2] + sqrdmulh V20.8H, V28.8H, V5.H[2] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V23.8H + sub V28.8H, V22.8H, V24.8H + add V21.8H, V21.8H, V23.8H + add V22.8H, V22.8H, V24.8H + mul V25.8H, V26.8H, V7.H[3] + mul V27.8H, V28.8H, V7.H[3] + sqrdmulh V23.8H, V26.8H, V5.H[3] + sqrdmulh V24.8H, V28.8H, V5.H[3] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V13.8H + sub V28.8H, V10.8H, V14.8H + add V9.8H, V9.8H, V13.8H + add V10.8H, V10.8H, V14.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V13.8H, V26.8H, V5.H[4] + sqrdmulh V14.8H, V28.8H, V5.H[4] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V13.8H, V13.8H, V25.8H + sub V14.8H, V14.8H, V27.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + sub V26.8H, V11.8H, V15.8H + sub V28.8H, V12.8H, V16.8H + add V11.8H, V11.8H, V15.8H + add V12.8H, V12.8H, V16.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V15.8H, V26.8H, V5.H[4] + sqrdmulh V16.8H, V28.8H, V5.H[4] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V27.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V21.8H + sub V28.8H, V18.8H, V22.8H + add V17.8H, V17.8H, V21.8H + add V18.8H, V18.8H, V22.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V21.8H, V26.8H, V5.H[5] + sqrdmulh V22.8H, V28.8H, V5.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V27.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V19.8H, V23.8H + sub V28.8H, V20.8H, V24.8H + add V19.8H, V19.8H, V23.8H + add V20.8H, V20.8H, V24.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V23.8H, V26.8H, V5.H[5] + sqrdmulh V24.8H, V28.8H, V5.H[5] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V10.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V10.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V11.8H, V8.H[2] + sqdmulh V26.8H, V12.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V11.8H, V25.8H, V8.H[0] + mls V12.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V18.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V18.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V19.8H, V8.H[2] + sqdmulh V26.8H, V20.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V19.8H, V25.8H, V8.H[0] + mls V20.8H, V26.8H, V8.H[0] + sub V26.8H, V9.8H, V17.8H + sub V28.8H, V10.8H, V18.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V17.8H, V26.8H, V5.H[6] + sqrdmulh V18.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V17.8H, V17.8H, V25.8H + sub V18.8H, V18.8H, V27.8H + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + sub V26.8H, V11.8H, V19.8H + sub V28.8H, V12.8H, V20.8H + add V11.8H, V11.8H, V19.8H + add V12.8H, V12.8H, V20.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V19.8H, V26.8H, V5.H[6] + sqrdmulh V20.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V27.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V13.8H, V21.8H + sub V28.8H, V14.8H, V22.8H + add V13.8H, V13.8H, V21.8H + add V14.8H, V14.8H, V22.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V21.8H, V26.8H, V5.H[6] + sqrdmulh V22.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V27.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V15.8H, V23.8H + sub V28.8H, V16.8H, V24.8H + add V15.8H, V15.8H, V23.8H + add V16.8H, V16.8H, V24.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V23.8H, V26.8H, V5.H[6] + sqrdmulh V24.8H, V28.8H, V5.H[6] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V27.8H, V27.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V27.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V25.8H, V9.8H, V7.H[7] + mul V26.8H, V10.8H, V7.H[7] + sqrdmulh V9.8H, V9.8H, V5.H[7] + sqrdmulh V10.8H, V10.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V9.8H, V9.8H, V25.8H + sub V10.8H, V10.8H, V26.8H + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V25.8H, V11.8H, V7.H[7] + mul V26.8H, V12.8H, V7.H[7] + sqrdmulh V11.8H, V11.8H, V5.H[7] + sqrdmulh V12.8H, V12.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V11.8H, V11.8H, V25.8H + sub V12.8H, V12.8H, V26.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V25.8H, V13.8H, V7.H[7] + mul V26.8H, V14.8H, V7.H[7] + sqrdmulh V13.8H, V13.8H, V5.H[7] + sqrdmulh V14.8H, V14.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V13.8H, V13.8H, V25.8H + sub V14.8H, V14.8H, V26.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V25.8H, V15.8H, V7.H[7] + mul V26.8H, V16.8H, V7.H[7] + sqrdmulh V15.8H, V15.8H, V5.H[7] + sqrdmulh V16.8H, V16.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V15.8H, V15.8H, V25.8H + sub V16.8H, V16.8H, V26.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + mul V25.8H, V17.8H, V7.H[7] + mul V26.8H, V18.8H, V7.H[7] + sqrdmulh V17.8H, V17.8H, V5.H[7] + sqrdmulh V18.8H, V18.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V17.8H, V17.8H, V25.8H + sub V18.8H, V18.8H, V26.8H + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + mul V25.8H, V19.8H, V7.H[7] + mul V26.8H, V20.8H, V7.H[7] + sqrdmulh V19.8H, V19.8H, V5.H[7] + sqrdmulh V20.8H, V20.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V19.8H, V19.8H, V25.8H + sub V20.8H, V20.8H, V26.8H + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + mul V25.8H, V21.8H, V7.H[7] + mul V26.8H, V22.8H, V7.H[7] + sqrdmulh V21.8H, V21.8H, V5.H[7] + sqrdmulh V22.8H, V22.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V21.8H, V21.8H, V25.8H + sub V22.8H, V22.8H, V26.8H + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V25.8H, V23.8H, V7.H[7] + mul V26.8H, V24.8H, V7.H[7] + sqrdmulh V23.8H, V23.8H, V5.H[7] + sqrdmulh V24.8H, V24.8H, V5.H[7] + sqrdmulh V25.8H, V25.8H, V8.H[0] + sqrdmulh V26.8H, V26.8H, V8.H[0] + sub V23.8H, V23.8H, V25.8H + sub V24.8H, V24.8H, V26.8H + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + str Q9, [x0, #16] + str Q10, [x0, #48] + str Q11, [x0, #80] + str Q12, [x0, #112] + str Q13, [x0, #144] + str Q14, [x0, #176] + str Q15, [x0, #208] + str Q16, [x0, #240] + str Q17, [x1, #16] + str Q18, [x1, #48] + str Q19, [x1, #80] + str Q20, [x1, #112] + str Q21, [x1, #144] + str Q22, [x1, #176] + str Q23, [x1, #208] + str Q24, [x1, #240] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -2886,10 +2886,10 @@ mlkem_invntt PROC mlkem_ntt_sqrdmlsh PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_zetas add x2, x2, L_mlkem_aarch64_zetas adrp x3, L_mlkem_aarch64_zetas_qinv @@ -2897,1143 +2897,1143 @@ mlkem_ntt_sqrdmlsh PROC adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts add x1, x0, #0x100 - ldr q4, [x4] - ldr q5, [x0] - ldr q6, [x0, #32] - ldr q7, [x0, #64] - ldr q8, [x0, #96] - ldr q9, [x0, #128] - ldr q10, [x0, #160] - ldr q11, [x0, #192] - ldr q12, [x0, #224] - ldr q13, [x1] - ldr q14, [x1, #32] - ldr q15, [x1, #64] - ldr q16, [x1, #96] - ldr q17, [x1, #128] - ldr q18, [x1, #160] - ldr q19, [x1, #192] - ldr q20, [x1, #224] - ldr q0, [x2] - ldr q1, [x3] - mul v29.8h, v13.8h, v1.h[1] - mul v30.8h, v14.8h, v1.h[1] - sqrdmulh v21.8h, v13.8h, v0.h[1] - sqrdmulh v22.8h, v14.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v15.8h, v1.h[1] - mul v30.8h, v16.8h, v1.h[1] - sqrdmulh v23.8h, v15.8h, v0.h[1] - sqrdmulh v24.8h, v16.8h, v0.h[1] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[1] - mul v30.8h, v18.8h, v1.h[1] - sqrdmulh v25.8h, v17.8h, v0.h[1] - sqrdmulh v26.8h, v18.8h, v0.h[1] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[1] - mul v30.8h, v20.8h, v1.h[1] - sqrdmulh v27.8h, v19.8h, v0.h[1] - sqrdmulh v28.8h, v20.8h, v0.h[1] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v13.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v14.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v15.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v16.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v9.8h, v25.8h - add v9.8h, v9.8h, v25.8h - sub v18.8h, v10.8h, v26.8h - add v10.8h, v10.8h, v26.8h - sub v19.8h, v11.8h, v27.8h - add v11.8h, v11.8h, v27.8h - sub v20.8h, v12.8h, v28.8h - add v12.8h, v12.8h, v28.8h - mul v29.8h, v9.8h, v1.h[2] - mul v30.8h, v10.8h, v1.h[2] - sqrdmulh v21.8h, v9.8h, v0.h[2] - sqrdmulh v22.8h, v10.8h, v0.h[2] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[2] - sqrdmulh v23.8h, v11.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[2] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[3] - mul v30.8h, v18.8h, v1.h[3] - sqrdmulh v25.8h, v17.8h, v0.h[3] - sqrdmulh v26.8h, v18.8h, v0.h[3] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[3] - mul v30.8h, v20.8h, v1.h[3] - sqrdmulh v27.8h, v19.8h, v0.h[3] - sqrdmulh v28.8h, v20.8h, v0.h[3] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v9.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v10.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v12.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v18.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v15.8h, v27.8h - add v15.8h, v15.8h, v27.8h - sub v20.8h, v16.8h, v28.8h - add v16.8h, v16.8h, v28.8h - mul v29.8h, v7.8h, v1.h[4] - mul v30.8h, v8.8h, v1.h[4] - sqrdmulh v21.8h, v7.8h, v0.h[4] - sqrdmulh v22.8h, v8.8h, v0.h[4] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[5] - mul v30.8h, v12.8h, v1.h[5] - sqrdmulh v23.8h, v11.8h, v0.h[5] - sqrdmulh v24.8h, v12.8h, v0.h[5] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v15.8h, v1.h[6] - mul v30.8h, v16.8h, v1.h[6] - sqrdmulh v25.8h, v15.8h, v0.h[6] - sqrdmulh v26.8h, v16.8h, v0.h[6] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[7] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v19.8h, v0.h[7] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v7.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v10.8h, v24.8h - add v10.8h, v10.8h, v24.8h - sub v15.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v18.8h, v28.8h - add v18.8h, v18.8h, v28.8h - ldr q0, [x2, #16] - ldr q1, [x3, #16] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - str q5, [x0] - str q6, [x0, #32] - str q7, [x0, #64] - str q8, [x0, #96] - str q9, [x0, #128] - str q10, [x0, #160] - str q11, [x0, #192] - str q12, [x0, #224] - str q13, [x1] - str q14, [x1, #32] - str q15, [x1, #64] - str q16, [x1, #96] - str q17, [x1, #128] - str q18, [x1, #160] - str q19, [x1, #192] - str q20, [x1, #224] - ldr q5, [x0, #16] - ldr q6, [x0, #48] - ldr q7, [x0, #80] - ldr q8, [x0, #112] - ldr q9, [x0, #144] - ldr q10, [x0, #176] - ldr q11, [x0, #208] - ldr q12, [x0, #240] - ldr q13, [x1, #16] - ldr q14, [x1, #48] - ldr q15, [x1, #80] - ldr q16, [x1, #112] - ldr q17, [x1, #144] - ldr q18, [x1, #176] - ldr q19, [x1, #208] - ldr q20, [x1, #240] - ldr q0, [x2] - ldr q1, [x3] - mul v29.8h, v13.8h, v1.h[1] - mul v30.8h, v14.8h, v1.h[1] - sqrdmulh v21.8h, v13.8h, v0.h[1] - sqrdmulh v22.8h, v14.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v15.8h, v1.h[1] - mul v30.8h, v16.8h, v1.h[1] - sqrdmulh v23.8h, v15.8h, v0.h[1] - sqrdmulh v24.8h, v16.8h, v0.h[1] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[1] - mul v30.8h, v18.8h, v1.h[1] - sqrdmulh v25.8h, v17.8h, v0.h[1] - sqrdmulh v26.8h, v18.8h, v0.h[1] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[1] - mul v30.8h, v20.8h, v1.h[1] - sqrdmulh v27.8h, v19.8h, v0.h[1] - sqrdmulh v28.8h, v20.8h, v0.h[1] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v13.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v14.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v15.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v16.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v9.8h, v25.8h - add v9.8h, v9.8h, v25.8h - sub v18.8h, v10.8h, v26.8h - add v10.8h, v10.8h, v26.8h - sub v19.8h, v11.8h, v27.8h - add v11.8h, v11.8h, v27.8h - sub v20.8h, v12.8h, v28.8h - add v12.8h, v12.8h, v28.8h - mul v29.8h, v9.8h, v1.h[2] - mul v30.8h, v10.8h, v1.h[2] - sqrdmulh v21.8h, v9.8h, v0.h[2] - sqrdmulh v22.8h, v10.8h, v0.h[2] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[2] - sqrdmulh v23.8h, v11.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[2] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v17.8h, v1.h[3] - mul v30.8h, v18.8h, v1.h[3] - sqrdmulh v25.8h, v17.8h, v0.h[3] - sqrdmulh v26.8h, v18.8h, v0.h[3] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[3] - mul v30.8h, v20.8h, v1.h[3] - sqrdmulh v27.8h, v19.8h, v0.h[3] - sqrdmulh v28.8h, v20.8h, v0.h[3] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v9.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v10.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v7.8h, v23.8h - add v7.8h, v7.8h, v23.8h - sub v12.8h, v8.8h, v24.8h - add v8.8h, v8.8h, v24.8h - sub v17.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v18.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v15.8h, v27.8h - add v15.8h, v15.8h, v27.8h - sub v20.8h, v16.8h, v28.8h - add v16.8h, v16.8h, v28.8h - mul v29.8h, v7.8h, v1.h[4] - mul v30.8h, v8.8h, v1.h[4] - sqrdmulh v21.8h, v7.8h, v0.h[4] - sqrdmulh v22.8h, v8.8h, v0.h[4] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v11.8h, v1.h[5] - mul v30.8h, v12.8h, v1.h[5] - sqrdmulh v23.8h, v11.8h, v0.h[5] - sqrdmulh v24.8h, v12.8h, v0.h[5] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v15.8h, v1.h[6] - mul v30.8h, v16.8h, v1.h[6] - sqrdmulh v25.8h, v15.8h, v0.h[6] - sqrdmulh v26.8h, v16.8h, v0.h[6] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v19.8h, v1.h[7] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v19.8h, v0.h[7] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v7.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v6.8h, v22.8h - add v6.8h, v6.8h, v22.8h - sub v11.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v10.8h, v24.8h - add v10.8h, v10.8h, v24.8h - sub v15.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v14.8h, v26.8h - add v14.8h, v14.8h, v26.8h - sub v19.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v18.8h, v28.8h - add v18.8h, v18.8h, v28.8h - ldr q0, [x2, #16] - ldr q1, [x3, #16] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - str q5, [x0, #16] - str q6, [x0, #48] - str q7, [x0, #80] - str q8, [x0, #112] - str q9, [x0, #144] - str q10, [x0, #176] - str q11, [x0, #208] - str q12, [x0, #240] - str q13, [x1, #16] - str q14, [x1, #48] - str q15, [x1, #80] - str q16, [x1, #112] - str q17, [x1, #144] - str q18, [x1, #176] - str q19, [x1, #208] - str q20, [x1, #240] - ldp q5, q6, [x0] - ldp q7, q8, [x0, #32] - ldp q9, q10, [x0, #64] - ldp q11, q12, [x0, #96] - ldp q13, q14, [x0, #128] - ldp q15, q16, [x0, #160] - ldp q17, q18, [x0, #192] - ldp q19, q20, [x0, #224] - ldr q0, [x2, #32] - ldr q1, [x3, #32] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #64] - ldr q2, [x2, #80] - ldr q1, [x3, #64] - ldr q3, [x3, #80] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.2d, v5.2d, v6.2d - trn1 v7.2d, v7.2d, v8.2d - trn2 v6.2d, v29.2d, v6.2d - trn2 v8.2d, v30.2d, v8.2d - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #96] - ldr q2, [x2, #112] - ldr q1, [x3, #96] - ldr q3, [x3, #112] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v29.2d, v10.2d - trn2 v12.2d, v30.2d, v12.2d - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #128] - ldr q2, [x2, #144] - ldr q1, [x3, #128] - ldr q3, [x3, #144] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v29.2d, v14.2d - trn2 v16.2d, v30.2d, v16.2d - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #160] - ldr q2, [x2, #176] - ldr q1, [x3, #160] - ldr q3, [x3, #176] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v29.2d, v18.2d - trn2 v20.2d, v30.2d, v20.2d - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #320] - ldr q2, [x2, #336] - ldr q1, [x3, #320] - ldr q3, [x3, #336] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.4s, v5.4s, v6.4s - trn1 v7.4s, v7.4s, v8.4s - trn2 v6.4s, v29.4s, v6.4s - trn2 v8.4s, v30.4s, v8.4s - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #352] - ldr q2, [x2, #368] - ldr q1, [x3, #352] - ldr q3, [x3, #368] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v29.4s, v10.4s - trn2 v12.4s, v30.4s, v12.4s - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #384] - ldr q2, [x2, #400] - ldr q1, [x3, #384] - ldr q3, [x3, #400] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v29.4s, v14.4s - trn2 v16.4s, v30.4s, v16.4s - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #416] - ldr q2, [x2, #432] - ldr q1, [x3, #416] - ldr q3, [x3, #432] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v29.4s, v18.4s - trn2 v20.4s, v30.4s, v20.4s - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - sqdmulh v21.8h, v5.8h, v4.h[2] - sqdmulh v22.8h, v6.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v5.8h, v21.8h, v4.h[0] - mls v6.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v7.8h, v4.h[2] - sqdmulh v22.8h, v8.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v7.8h, v21.8h, v4.h[0] - mls v8.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v9.8h, v4.h[2] - sqdmulh v22.8h, v10.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v9.8h, v21.8h, v4.h[0] - mls v10.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v11.8h, v4.h[2] - sqdmulh v22.8h, v12.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v11.8h, v21.8h, v4.h[0] - mls v12.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v13.8h, v4.h[2] - sqdmulh v22.8h, v14.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v13.8h, v21.8h, v4.h[0] - mls v14.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v15.8h, v4.h[2] - sqdmulh v22.8h, v16.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v15.8h, v21.8h, v4.h[0] - mls v16.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v17.8h, v4.h[2] - sqdmulh v22.8h, v18.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v17.8h, v21.8h, v4.h[0] - mls v18.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v19.8h, v4.h[2] - sqdmulh v22.8h, v20.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v19.8h, v21.8h, v4.h[0] - mls v20.8h, v22.8h, v4.h[0] - mov v29.16b, v5.16b - trn1 v5.4s, v5.4s, v6.4s - trn2 v6.4s, v29.4s, v6.4s - mov v29.16b, v5.16b - trn1 v5.2d, v5.2d, v6.2d - trn2 v6.2d, v29.2d, v6.2d - mov v29.16b, v7.16b - trn1 v7.4s, v7.4s, v8.4s - trn2 v8.4s, v29.4s, v8.4s - mov v29.16b, v7.16b - trn1 v7.2d, v7.2d, v8.2d - trn2 v8.2d, v29.2d, v8.2d - mov v29.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v29.4s, v10.4s - mov v29.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v29.2d, v10.2d - mov v29.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v29.4s, v12.4s - mov v29.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v29.2d, v12.2d - mov v29.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v29.4s, v14.4s - mov v29.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v29.2d, v14.2d - mov v29.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v29.4s, v16.4s - mov v29.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v29.2d, v16.2d - mov v29.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v29.4s, v18.4s - mov v29.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v29.2d, v18.2d - mov v29.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v29.4s, v20.4s - mov v29.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v29.2d, v20.2d - stp q5, q6, [x0] - stp q7, q8, [x0, #32] - stp q9, q10, [x0, #64] - stp q11, q12, [x0, #96] - stp q13, q14, [x0, #128] - stp q15, q16, [x0, #160] - stp q17, q18, [x0, #192] - stp q19, q20, [x0, #224] - ldp q5, q6, [x1] - ldp q7, q8, [x1, #32] - ldp q9, q10, [x1, #64] - ldp q11, q12, [x1, #96] - ldp q13, q14, [x1, #128] - ldp q15, q16, [x1, #160] - ldp q17, q18, [x1, #192] - ldp q19, q20, [x1, #224] - ldr q0, [x2, #48] - ldr q1, [x3, #48] - mul v29.8h, v6.8h, v1.h[0] - mul v30.8h, v8.8h, v1.h[1] - sqrdmulh v21.8h, v6.8h, v0.h[0] - sqrdmulh v22.8h, v8.8h, v0.h[1] - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v29.8h, v10.8h, v1.h[2] - mul v30.8h, v12.8h, v1.h[3] - sqrdmulh v23.8h, v10.8h, v0.h[2] - sqrdmulh v24.8h, v12.8h, v0.h[3] - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v29.8h, v14.8h, v1.h[4] - mul v30.8h, v16.8h, v1.h[5] - sqrdmulh v25.8h, v14.8h, v0.h[4] - sqrdmulh v26.8h, v16.8h, v0.h[5] - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - mul v29.8h, v18.8h, v1.h[6] - mul v30.8h, v20.8h, v1.h[7] - sqrdmulh v27.8h, v18.8h, v0.h[6] - sqrdmulh v28.8h, v20.8h, v0.h[7] - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #192] - ldr q2, [x2, #208] - ldr q1, [x3, #192] - ldr q3, [x3, #208] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.2d, v5.2d, v6.2d - trn1 v7.2d, v7.2d, v8.2d - trn2 v6.2d, v29.2d, v6.2d - trn2 v8.2d, v30.2d, v8.2d - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #224] - ldr q2, [x2, #240] - ldr q1, [x3, #224] - ldr q3, [x3, #240] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v29.2d, v10.2d - trn2 v12.2d, v30.2d, v12.2d - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #256] - ldr q2, [x2, #272] - ldr q1, [x3, #256] - ldr q3, [x3, #272] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v29.2d, v14.2d - trn2 v16.2d, v30.2d, v16.2d - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #288] - ldr q2, [x2, #304] - ldr q1, [x3, #288] - ldr q3, [x3, #304] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v29.2d, v18.2d - trn2 v20.2d, v30.2d, v20.2d - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - ldr q0, [x2, #448] - ldr q2, [x2, #464] - ldr q1, [x3, #448] - ldr q3, [x3, #464] - mov v29.16b, v5.16b - mov v30.16b, v7.16b - trn1 v5.4s, v5.4s, v6.4s - trn1 v7.4s, v7.4s, v8.4s - trn2 v6.4s, v29.4s, v6.4s - trn2 v8.4s, v30.4s, v8.4s - mul v29.8h, v6.8h, v1.8h - mul v30.8h, v8.8h, v3.8h - sqrdmulh v21.8h, v6.8h, v0.8h - sqrdmulh v22.8h, v8.8h, v2.8h - sqrdmlsh v21.8h, v29.8h, v4.h[0] - sqrdmlsh v22.8h, v30.8h, v4.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - ldr q0, [x2, #480] - ldr q2, [x2, #496] - ldr q1, [x3, #480] - ldr q3, [x3, #496] - mov v29.16b, v9.16b - mov v30.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v29.4s, v10.4s - trn2 v12.4s, v30.4s, v12.4s - mul v29.8h, v10.8h, v1.8h - mul v30.8h, v12.8h, v3.8h - sqrdmulh v23.8h, v10.8h, v0.8h - sqrdmulh v24.8h, v12.8h, v2.8h - sqrdmlsh v23.8h, v29.8h, v4.h[0] - sqrdmlsh v24.8h, v30.8h, v4.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #512] - ldr q2, [x2, #528] - ldr q1, [x3, #512] - ldr q3, [x3, #528] - mov v29.16b, v13.16b - mov v30.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v29.4s, v14.4s - trn2 v16.4s, v30.4s, v16.4s - mul v29.8h, v14.8h, v1.8h - mul v30.8h, v16.8h, v3.8h - sqrdmulh v25.8h, v14.8h, v0.8h - sqrdmulh v26.8h, v16.8h, v2.8h - sqrdmlsh v25.8h, v29.8h, v4.h[0] - sqrdmlsh v26.8h, v30.8h, v4.h[0] - sshr v25.8h, v25.8h, #1 - sshr v26.8h, v26.8h, #1 - ldr q0, [x2, #544] - ldr q2, [x2, #560] - ldr q1, [x3, #544] - ldr q3, [x3, #560] - mov v29.16b, v17.16b - mov v30.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v29.4s, v18.4s - trn2 v20.4s, v30.4s, v20.4s - mul v29.8h, v18.8h, v1.8h - mul v30.8h, v20.8h, v3.8h - sqrdmulh v27.8h, v18.8h, v0.8h - sqrdmulh v28.8h, v20.8h, v2.8h - sqrdmlsh v27.8h, v29.8h, v4.h[0] - sqrdmlsh v28.8h, v30.8h, v4.h[0] - sshr v27.8h, v27.8h, #1 - sshr v28.8h, v28.8h, #1 - sub v6.8h, v5.8h, v21.8h - add v5.8h, v5.8h, v21.8h - sub v8.8h, v7.8h, v22.8h - add v7.8h, v7.8h, v22.8h - sub v10.8h, v9.8h, v23.8h - add v9.8h, v9.8h, v23.8h - sub v12.8h, v11.8h, v24.8h - add v11.8h, v11.8h, v24.8h - sub v14.8h, v13.8h, v25.8h - add v13.8h, v13.8h, v25.8h - sub v16.8h, v15.8h, v26.8h - add v15.8h, v15.8h, v26.8h - sub v18.8h, v17.8h, v27.8h - add v17.8h, v17.8h, v27.8h - sub v20.8h, v19.8h, v28.8h - add v19.8h, v19.8h, v28.8h - sqdmulh v21.8h, v5.8h, v4.h[2] - sqdmulh v22.8h, v6.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v5.8h, v21.8h, v4.h[0] - mls v6.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v7.8h, v4.h[2] - sqdmulh v22.8h, v8.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v7.8h, v21.8h, v4.h[0] - mls v8.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v9.8h, v4.h[2] - sqdmulh v22.8h, v10.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v9.8h, v21.8h, v4.h[0] - mls v10.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v11.8h, v4.h[2] - sqdmulh v22.8h, v12.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v11.8h, v21.8h, v4.h[0] - mls v12.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v13.8h, v4.h[2] - sqdmulh v22.8h, v14.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v13.8h, v21.8h, v4.h[0] - mls v14.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v15.8h, v4.h[2] - sqdmulh v22.8h, v16.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v15.8h, v21.8h, v4.h[0] - mls v16.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v17.8h, v4.h[2] - sqdmulh v22.8h, v18.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v17.8h, v21.8h, v4.h[0] - mls v18.8h, v22.8h, v4.h[0] - sqdmulh v21.8h, v19.8h, v4.h[2] - sqdmulh v22.8h, v20.8h, v4.h[2] - sshr v21.8h, v21.8h, #11 - sshr v22.8h, v22.8h, #11 - mls v19.8h, v21.8h, v4.h[0] - mls v20.8h, v22.8h, v4.h[0] - mov v29.16b, v5.16b - trn1 v5.4s, v5.4s, v6.4s - trn2 v6.4s, v29.4s, v6.4s - mov v29.16b, v5.16b - trn1 v5.2d, v5.2d, v6.2d - trn2 v6.2d, v29.2d, v6.2d - mov v29.16b, v7.16b - trn1 v7.4s, v7.4s, v8.4s - trn2 v8.4s, v29.4s, v8.4s - mov v29.16b, v7.16b - trn1 v7.2d, v7.2d, v8.2d - trn2 v8.2d, v29.2d, v8.2d - mov v29.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v29.4s, v10.4s - mov v29.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v29.2d, v10.2d - mov v29.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v29.4s, v12.4s - mov v29.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v29.2d, v12.2d - mov v29.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v29.4s, v14.4s - mov v29.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v29.2d, v14.2d - mov v29.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v29.4s, v16.4s - mov v29.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v29.2d, v16.2d - mov v29.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v29.4s, v18.4s - mov v29.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v29.2d, v18.2d - mov v29.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v29.4s, v20.4s - mov v29.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v29.2d, v20.2d - stp q5, q6, [x1] - stp q7, q8, [x1, #32] - stp q9, q10, [x1, #64] - stp q11, q12, [x1, #96] - stp q13, q14, [x1, #128] - stp q15, q16, [x1, #160] - stp q17, q18, [x1, #192] - stp q19, q20, [x1, #224] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q4, [x4] + ldr Q5, [x0] + ldr Q6, [x0, #32] + ldr Q7, [x0, #64] + ldr Q8, [x0, #96] + ldr Q9, [x0, #128] + ldr Q10, [x0, #160] + ldr Q11, [x0, #192] + ldr Q12, [x0, #224] + ldr Q13, [x1] + ldr Q14, [x1, #32] + ldr Q15, [x1, #64] + ldr Q16, [x1, #96] + ldr Q17, [x1, #128] + ldr Q18, [x1, #160] + ldr Q19, [x1, #192] + ldr Q20, [x1, #224] + ldr Q0, [x2] + ldr Q1, [x3] + mul V29.8H, V13.8H, V1.H[1] + mul V30.8H, V14.8H, V1.H[1] + sqrdmulh V21.8H, V13.8H, V0.H[1] + sqrdmulh V22.8H, V14.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V15.8H, V1.H[1] + mul V30.8H, V16.8H, V1.H[1] + sqrdmulh V23.8H, V15.8H, V0.H[1] + sqrdmulh V24.8H, V16.8H, V0.H[1] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[1] + mul V30.8H, V18.8H, V1.H[1] + sqrdmulh V25.8H, V17.8H, V0.H[1] + sqrdmulh V26.8H, V18.8H, V0.H[1] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[1] + mul V30.8H, V20.8H, V1.H[1] + sqrdmulh V27.8H, V19.8H, V0.H[1] + sqrdmulh V28.8H, V20.8H, V0.H[1] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V13.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V14.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V15.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V16.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V9.8H, V25.8H + add V9.8H, V9.8H, V25.8H + sub V18.8H, V10.8H, V26.8H + add V10.8H, V10.8H, V26.8H + sub V19.8H, V11.8H, V27.8H + add V11.8H, V11.8H, V27.8H + sub V20.8H, V12.8H, V28.8H + add V12.8H, V12.8H, V28.8H + mul V29.8H, V9.8H, V1.H[2] + mul V30.8H, V10.8H, V1.H[2] + sqrdmulh V21.8H, V9.8H, V0.H[2] + sqrdmulh V22.8H, V10.8H, V0.H[2] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[2] + sqrdmulh V23.8H, V11.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[2] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[3] + mul V30.8H, V18.8H, V1.H[3] + sqrdmulh V25.8H, V17.8H, V0.H[3] + sqrdmulh V26.8H, V18.8H, V0.H[3] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[3] + mul V30.8H, V20.8H, V1.H[3] + sqrdmulh V27.8H, V19.8H, V0.H[3] + sqrdmulh V28.8H, V20.8H, V0.H[3] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V9.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V10.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V12.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V18.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V15.8H, V27.8H + add V15.8H, V15.8H, V27.8H + sub V20.8H, V16.8H, V28.8H + add V16.8H, V16.8H, V28.8H + mul V29.8H, V7.8H, V1.H[4] + mul V30.8H, V8.8H, V1.H[4] + sqrdmulh V21.8H, V7.8H, V0.H[4] + sqrdmulh V22.8H, V8.8H, V0.H[4] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[5] + mul V30.8H, V12.8H, V1.H[5] + sqrdmulh V23.8H, V11.8H, V0.H[5] + sqrdmulh V24.8H, V12.8H, V0.H[5] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V15.8H, V1.H[6] + mul V30.8H, V16.8H, V1.H[6] + sqrdmulh V25.8H, V15.8H, V0.H[6] + sqrdmulh V26.8H, V16.8H, V0.H[6] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[7] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V19.8H, V0.H[7] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V7.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V10.8H, V24.8H + add V10.8H, V10.8H, V24.8H + sub V15.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V18.8H, V28.8H + add V18.8H, V18.8H, V28.8H + ldr Q0, [x2, #16] + ldr Q1, [x3, #16] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + str Q5, [x0] + str Q6, [x0, #32] + str Q7, [x0, #64] + str Q8, [x0, #96] + str Q9, [x0, #128] + str Q10, [x0, #160] + str Q11, [x0, #192] + str Q12, [x0, #224] + str Q13, [x1] + str Q14, [x1, #32] + str Q15, [x1, #64] + str Q16, [x1, #96] + str Q17, [x1, #128] + str Q18, [x1, #160] + str Q19, [x1, #192] + str Q20, [x1, #224] + ldr Q5, [x0, #16] + ldr Q6, [x0, #48] + ldr Q7, [x0, #80] + ldr Q8, [x0, #112] + ldr Q9, [x0, #144] + ldr Q10, [x0, #176] + ldr Q11, [x0, #208] + ldr Q12, [x0, #240] + ldr Q13, [x1, #16] + ldr Q14, [x1, #48] + ldr Q15, [x1, #80] + ldr Q16, [x1, #112] + ldr Q17, [x1, #144] + ldr Q18, [x1, #176] + ldr Q19, [x1, #208] + ldr Q20, [x1, #240] + ldr Q0, [x2] + ldr Q1, [x3] + mul V29.8H, V13.8H, V1.H[1] + mul V30.8H, V14.8H, V1.H[1] + sqrdmulh V21.8H, V13.8H, V0.H[1] + sqrdmulh V22.8H, V14.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V15.8H, V1.H[1] + mul V30.8H, V16.8H, V1.H[1] + sqrdmulh V23.8H, V15.8H, V0.H[1] + sqrdmulh V24.8H, V16.8H, V0.H[1] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[1] + mul V30.8H, V18.8H, V1.H[1] + sqrdmulh V25.8H, V17.8H, V0.H[1] + sqrdmulh V26.8H, V18.8H, V0.H[1] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[1] + mul V30.8H, V20.8H, V1.H[1] + sqrdmulh V27.8H, V19.8H, V0.H[1] + sqrdmulh V28.8H, V20.8H, V0.H[1] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V13.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V14.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V15.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V16.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V9.8H, V25.8H + add V9.8H, V9.8H, V25.8H + sub V18.8H, V10.8H, V26.8H + add V10.8H, V10.8H, V26.8H + sub V19.8H, V11.8H, V27.8H + add V11.8H, V11.8H, V27.8H + sub V20.8H, V12.8H, V28.8H + add V12.8H, V12.8H, V28.8H + mul V29.8H, V9.8H, V1.H[2] + mul V30.8H, V10.8H, V1.H[2] + sqrdmulh V21.8H, V9.8H, V0.H[2] + sqrdmulh V22.8H, V10.8H, V0.H[2] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[2] + sqrdmulh V23.8H, V11.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[2] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V17.8H, V1.H[3] + mul V30.8H, V18.8H, V1.H[3] + sqrdmulh V25.8H, V17.8H, V0.H[3] + sqrdmulh V26.8H, V18.8H, V0.H[3] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[3] + mul V30.8H, V20.8H, V1.H[3] + sqrdmulh V27.8H, V19.8H, V0.H[3] + sqrdmulh V28.8H, V20.8H, V0.H[3] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V9.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V10.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V7.8H, V23.8H + add V7.8H, V7.8H, V23.8H + sub V12.8H, V8.8H, V24.8H + add V8.8H, V8.8H, V24.8H + sub V17.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V18.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V15.8H, V27.8H + add V15.8H, V15.8H, V27.8H + sub V20.8H, V16.8H, V28.8H + add V16.8H, V16.8H, V28.8H + mul V29.8H, V7.8H, V1.H[4] + mul V30.8H, V8.8H, V1.H[4] + sqrdmulh V21.8H, V7.8H, V0.H[4] + sqrdmulh V22.8H, V8.8H, V0.H[4] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V11.8H, V1.H[5] + mul V30.8H, V12.8H, V1.H[5] + sqrdmulh V23.8H, V11.8H, V0.H[5] + sqrdmulh V24.8H, V12.8H, V0.H[5] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V15.8H, V1.H[6] + mul V30.8H, V16.8H, V1.H[6] + sqrdmulh V25.8H, V15.8H, V0.H[6] + sqrdmulh V26.8H, V16.8H, V0.H[6] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V19.8H, V1.H[7] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V19.8H, V0.H[7] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V7.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V6.8H, V22.8H + add V6.8H, V6.8H, V22.8H + sub V11.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V10.8H, V24.8H + add V10.8H, V10.8H, V24.8H + sub V15.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V14.8H, V26.8H + add V14.8H, V14.8H, V26.8H + sub V19.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V18.8H, V28.8H + add V18.8H, V18.8H, V28.8H + ldr Q0, [x2, #16] + ldr Q1, [x3, #16] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + str Q5, [x0, #16] + str Q6, [x0, #48] + str Q7, [x0, #80] + str Q8, [x0, #112] + str Q9, [x0, #144] + str Q10, [x0, #176] + str Q11, [x0, #208] + str Q12, [x0, #240] + str Q13, [x1, #16] + str Q14, [x1, #48] + str Q15, [x1, #80] + str Q16, [x1, #112] + str Q17, [x1, #144] + str Q18, [x1, #176] + str Q19, [x1, #208] + str Q20, [x1, #240] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x2, #32] + ldr Q1, [x3, #32] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #64] + ldr Q2, [x2, #80] + ldr Q1, [x3, #64] + ldr Q3, [x3, #80] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #96] + ldr Q2, [x2, #112] + ldr Q1, [x3, #96] + ldr Q3, [x3, #112] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #128] + ldr Q2, [x2, #144] + ldr Q1, [x3, #128] + ldr Q3, [x3, #144] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #160] + ldr Q2, [x2, #176] + ldr Q1, [x3, #160] + ldr Q3, [x3, #176] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #320] + ldr Q2, [x2, #336] + ldr Q1, [x3, #320] + ldr Q3, [x3, #336] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #352] + ldr Q2, [x2, #368] + ldr Q1, [x3, #352] + ldr Q3, [x3, #368] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #384] + ldr Q2, [x2, #400] + ldr Q1, [x3, #384] + ldr Q3, [x3, #400] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #416] + ldr Q2, [x2, #432] + ldr Q1, [x3, #416] + ldr Q3, [x3, #432] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + sqdmulh V21.8H, V5.8H, V4.H[2] + sqdmulh V22.8H, V6.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V5.8H, V21.8H, V4.H[0] + mls V6.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V7.8H, V4.H[2] + sqdmulh V22.8H, V8.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V7.8H, V21.8H, V4.H[0] + mls V8.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V9.8H, V4.H[2] + sqdmulh V22.8H, V10.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V9.8H, V21.8H, V4.H[0] + mls V10.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V11.8H, V4.H[2] + sqdmulh V22.8H, V12.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V11.8H, V21.8H, V4.H[0] + mls V12.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V13.8H, V4.H[2] + sqdmulh V22.8H, V14.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V13.8H, V21.8H, V4.H[0] + mls V14.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V15.8H, V4.H[2] + sqdmulh V22.8H, V16.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V15.8H, V21.8H, V4.H[0] + mls V16.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V17.8H, V4.H[2] + sqdmulh V22.8H, V18.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V17.8H, V21.8H, V4.H[0] + mls V18.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V19.8H, V4.H[2] + sqdmulh V22.8H, V20.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V19.8H, V21.8H, V4.H[0] + mls V20.8H, V22.8H, V4.H[0] + mov V29.16B, V5.16B + trn1 V5.4S, V5.4S, V6.4S + trn2 V6.4S, V29.4S, V6.4S + mov V29.16B, V5.16B + trn1 V5.2D, V5.2D, V6.2D + trn2 V6.2D, V29.2D, V6.2D + mov V29.16B, V7.16B + trn1 V7.4S, V7.4S, V8.4S + trn2 V8.4S, V29.4S, V8.4S + mov V29.16B, V7.16B + trn1 V7.2D, V7.2D, V8.2D + trn2 V8.2D, V29.2D, V8.2D + mov V29.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V29.4S, V10.4S + mov V29.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V29.2D, V10.2D + mov V29.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V29.4S, V12.4S + mov V29.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V29.2D, V12.2D + mov V29.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V29.4S, V14.4S + mov V29.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V29.2D, V14.2D + mov V29.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V29.4S, V16.4S + mov V29.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V29.2D, V16.2D + mov V29.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V29.4S, V18.4S + mov V29.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V29.2D, V18.2D + mov V29.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V29.4S, V20.4S + mov V29.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V29.2D, V20.2D + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x1] + ldp Q7, Q8, [x1, #32] + ldp Q9, Q10, [x1, #64] + ldp Q11, Q12, [x1, #96] + ldp Q13, Q14, [x1, #128] + ldp Q15, Q16, [x1, #160] + ldp Q17, Q18, [x1, #192] + ldp Q19, Q20, [x1, #224] + ldr Q0, [x2, #48] + ldr Q1, [x3, #48] + mul V29.8H, V6.8H, V1.H[0] + mul V30.8H, V8.8H, V1.H[1] + sqrdmulh V21.8H, V6.8H, V0.H[0] + sqrdmulh V22.8H, V8.8H, V0.H[1] + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V29.8H, V10.8H, V1.H[2] + mul V30.8H, V12.8H, V1.H[3] + sqrdmulh V23.8H, V10.8H, V0.H[2] + sqrdmulh V24.8H, V12.8H, V0.H[3] + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V29.8H, V14.8H, V1.H[4] + mul V30.8H, V16.8H, V1.H[5] + sqrdmulh V25.8H, V14.8H, V0.H[4] + sqrdmulh V26.8H, V16.8H, V0.H[5] + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + mul V29.8H, V18.8H, V1.H[6] + mul V30.8H, V20.8H, V1.H[7] + sqrdmulh V27.8H, V18.8H, V0.H[6] + sqrdmulh V28.8H, V20.8H, V0.H[7] + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #192] + ldr Q2, [x2, #208] + ldr Q1, [x3, #192] + ldr Q3, [x3, #208] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #224] + ldr Q2, [x2, #240] + ldr Q1, [x3, #224] + ldr Q3, [x3, #240] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #256] + ldr Q2, [x2, #272] + ldr Q1, [x3, #256] + ldr Q3, [x3, #272] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #288] + ldr Q2, [x2, #304] + ldr Q1, [x3, #288] + ldr Q3, [x3, #304] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + ldr Q0, [x2, #448] + ldr Q2, [x2, #464] + ldr Q1, [x3, #448] + ldr Q3, [x3, #464] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.8H, V6.8H, V1.8H + mul V30.8H, V8.8H, V3.8H + sqrdmulh V21.8H, V6.8H, V0.8H + sqrdmulh V22.8H, V8.8H, V2.8H + sqrdmlsh V21.8H, V29.8H, V4.H[0] + sqrdmlsh V22.8H, V30.8H, V4.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + ldr Q0, [x2, #480] + ldr Q2, [x2, #496] + ldr Q1, [x3, #480] + ldr Q3, [x3, #496] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.8H, V10.8H, V1.8H + mul V30.8H, V12.8H, V3.8H + sqrdmulh V23.8H, V10.8H, V0.8H + sqrdmulh V24.8H, V12.8H, V2.8H + sqrdmlsh V23.8H, V29.8H, V4.H[0] + sqrdmlsh V24.8H, V30.8H, V4.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #512] + ldr Q2, [x2, #528] + ldr Q1, [x3, #512] + ldr Q3, [x3, #528] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.8H, V14.8H, V1.8H + mul V30.8H, V16.8H, V3.8H + sqrdmulh V25.8H, V14.8H, V0.8H + sqrdmulh V26.8H, V16.8H, V2.8H + sqrdmlsh V25.8H, V29.8H, V4.H[0] + sqrdmlsh V26.8H, V30.8H, V4.H[0] + sshr V25.8H, V25.8H, #1 + sshr V26.8H, V26.8H, #1 + ldr Q0, [x2, #544] + ldr Q2, [x2, #560] + ldr Q1, [x3, #544] + ldr Q3, [x3, #560] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.8H, V18.8H, V1.8H + mul V30.8H, V20.8H, V3.8H + sqrdmulh V27.8H, V18.8H, V0.8H + sqrdmulh V28.8H, V20.8H, V2.8H + sqrdmlsh V27.8H, V29.8H, V4.H[0] + sqrdmlsh V28.8H, V30.8H, V4.H[0] + sshr V27.8H, V27.8H, #1 + sshr V28.8H, V28.8H, #1 + sub V6.8H, V5.8H, V21.8H + add V5.8H, V5.8H, V21.8H + sub V8.8H, V7.8H, V22.8H + add V7.8H, V7.8H, V22.8H + sub V10.8H, V9.8H, V23.8H + add V9.8H, V9.8H, V23.8H + sub V12.8H, V11.8H, V24.8H + add V11.8H, V11.8H, V24.8H + sub V14.8H, V13.8H, V25.8H + add V13.8H, V13.8H, V25.8H + sub V16.8H, V15.8H, V26.8H + add V15.8H, V15.8H, V26.8H + sub V18.8H, V17.8H, V27.8H + add V17.8H, V17.8H, V27.8H + sub V20.8H, V19.8H, V28.8H + add V19.8H, V19.8H, V28.8H + sqdmulh V21.8H, V5.8H, V4.H[2] + sqdmulh V22.8H, V6.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V5.8H, V21.8H, V4.H[0] + mls V6.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V7.8H, V4.H[2] + sqdmulh V22.8H, V8.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V7.8H, V21.8H, V4.H[0] + mls V8.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V9.8H, V4.H[2] + sqdmulh V22.8H, V10.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V9.8H, V21.8H, V4.H[0] + mls V10.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V11.8H, V4.H[2] + sqdmulh V22.8H, V12.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V11.8H, V21.8H, V4.H[0] + mls V12.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V13.8H, V4.H[2] + sqdmulh V22.8H, V14.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V13.8H, V21.8H, V4.H[0] + mls V14.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V15.8H, V4.H[2] + sqdmulh V22.8H, V16.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V15.8H, V21.8H, V4.H[0] + mls V16.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V17.8H, V4.H[2] + sqdmulh V22.8H, V18.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V17.8H, V21.8H, V4.H[0] + mls V18.8H, V22.8H, V4.H[0] + sqdmulh V21.8H, V19.8H, V4.H[2] + sqdmulh V22.8H, V20.8H, V4.H[2] + sshr V21.8H, V21.8H, #11 + sshr V22.8H, V22.8H, #11 + mls V19.8H, V21.8H, V4.H[0] + mls V20.8H, V22.8H, V4.H[0] + mov V29.16B, V5.16B + trn1 V5.4S, V5.4S, V6.4S + trn2 V6.4S, V29.4S, V6.4S + mov V29.16B, V5.16B + trn1 V5.2D, V5.2D, V6.2D + trn2 V6.2D, V29.2D, V6.2D + mov V29.16B, V7.16B + trn1 V7.4S, V7.4S, V8.4S + trn2 V8.4S, V29.4S, V8.4S + mov V29.16B, V7.16B + trn1 V7.2D, V7.2D, V8.2D + trn2 V8.2D, V29.2D, V8.2D + mov V29.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V29.4S, V10.4S + mov V29.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V29.2D, V10.2D + mov V29.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V29.4S, V12.4S + mov V29.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V29.2D, V12.2D + mov V29.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V29.4S, V14.4S + mov V29.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V29.2D, V14.2D + mov V29.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V29.4S, V16.4S + mov V29.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V29.2D, V16.2D + mov V29.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V29.4S, V18.4S + mov V29.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V29.2D, V18.2D + mov V29.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V29.4S, V20.4S + mov V29.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V29.2D, V20.2D + stp Q5, Q6, [x1] + stp Q7, Q8, [x1, #32] + stp Q9, Q10, [x1, #64] + stp Q11, Q12, [x1, #96] + stp Q13, Q14, [x1, #128] + stp Q15, Q16, [x1, #160] + stp Q17, Q18, [x1, #192] + stp Q19, Q20, [x1, #224] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -4043,10 +4043,10 @@ mlkem_ntt_sqrdmlsh PROC mlkem_invntt_sqrdmlsh PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_zetas_inv add x2, x2, L_mlkem_aarch64_zetas_inv adrp x3, L_mlkem_aarch64_zetas_inv_qinv @@ -4054,1272 +4054,1272 @@ mlkem_invntt_sqrdmlsh PROC adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts add x1, x0, #0x100 - ldr q8, [x4] - ldp q9, q10, [x0] - ldp q11, q12, [x0, #32] - ldp q13, q14, [x0, #64] - ldp q15, q16, [x0, #96] - ldp q17, q18, [x0, #128] - ldp q19, q20, [x0, #160] - ldp q21, q22, [x0, #192] - ldp q23, q24, [x0, #224] - mov v25.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v25.2d, v10.2d - mov v25.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v25.4s, v10.4s - mov v25.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v25.2d, v12.2d - mov v25.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v25.4s, v12.4s - mov v25.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v25.2d, v14.2d - mov v25.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v25.4s, v14.4s - mov v25.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v25.2d, v16.2d - mov v25.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v25.4s, v16.4s - mov v25.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v25.2d, v18.2d - mov v25.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v25.4s, v18.4s - mov v25.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v25.2d, v20.2d - mov v25.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v25.4s, v20.4s - mov v25.16b, v21.16b - trn1 v21.2d, v21.2d, v22.2d - trn2 v22.2d, v25.2d, v22.2d - mov v25.16b, v21.16b - trn1 v21.4s, v21.4s, v22.4s - trn2 v22.4s, v25.4s, v22.4s - mov v25.16b, v23.16b - trn1 v23.2d, v23.2d, v24.2d - trn2 v24.2d, v25.2d, v24.2d - mov v25.16b, v23.16b - trn1 v23.4s, v23.4s, v24.4s - trn2 v24.4s, v25.4s, v24.4s - ldr q0, [x2] - ldr q1, [x2, #16] - ldr q2, [x3] - ldr q3, [x3, #16] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #32] - ldr q1, [x2, #48] - ldr q2, [x3, #32] - ldr q3, [x3, #48] - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #64] - ldr q1, [x2, #80] - ldr q2, [x3, #64] - ldr q3, [x3, #80] - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #96] - ldr q1, [x2, #112] - ldr q2, [x3, #96] - ldr q3, [x3, #112] - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #256] - ldr q1, [x2, #272] - ldr q2, [x3, #256] - ldr q3, [x3, #272] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v25.4s, v10.4s - trn2 v12.4s, v26.4s, v12.4s - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #288] - ldr q1, [x2, #304] - ldr q2, [x3, #288] - ldr q3, [x3, #304] - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v25.4s, v14.4s - trn2 v16.4s, v26.4s, v16.4s - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #320] - ldr q1, [x2, #336] - ldr q2, [x3, #320] - ldr q3, [x3, #336] - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v25.4s, v18.4s - trn2 v20.4s, v26.4s, v20.4s - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #352] - ldr q1, [x2, #368] - ldr q2, [x3, #352] - ldr q3, [x3, #368] - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.4s, v21.4s, v22.4s - trn1 v23.4s, v23.4s, v24.4s - trn2 v22.4s, v25.4s, v22.4s - trn2 v24.4s, v26.4s, v24.4s - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #512] - ldr q2, [x3, #512] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v25.2d, v10.2d - trn2 v12.2d, v26.2d, v12.2d - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.h[0] - mul v27.8h, v28.8h, v2.h[1] - sqrdmulh v10.8h, v26.8h, v0.h[0] - sqrdmulh v12.8h, v28.8h, v0.h[1] - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v25.2d, v14.2d - trn2 v16.2d, v26.2d, v16.2d - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.h[2] - mul v27.8h, v28.8h, v2.h[3] - sqrdmulh v14.8h, v26.8h, v0.h[2] - sqrdmulh v16.8h, v28.8h, v0.h[3] - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v25.2d, v18.2d - trn2 v20.2d, v26.2d, v20.2d - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.h[4] - mul v27.8h, v28.8h, v2.h[5] - sqrdmulh v18.8h, v26.8h, v0.h[4] - sqrdmulh v20.8h, v28.8h, v0.h[5] - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.2d, v21.2d, v22.2d - trn1 v23.2d, v23.2d, v24.2d - trn2 v22.2d, v25.2d, v22.2d - trn2 v24.2d, v26.2d, v24.2d - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.h[6] - mul v27.8h, v28.8h, v2.h[7] - sqrdmulh v22.8h, v26.8h, v0.h[6] - sqrdmulh v24.8h, v28.8h, v0.h[7] - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v11.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v11.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v13.8h, v8.h[2] - sqdmulh v26.8h, v15.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v13.8h, v25.8h, v8.h[0] - mls v15.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v19.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v19.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v21.8h, v8.h[2] - sqdmulh v26.8h, v23.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v21.8h, v25.8h, v8.h[0] - mls v23.8h, v26.8h, v8.h[0] - stp q9, q10, [x0] - stp q11, q12, [x0, #32] - stp q13, q14, [x0, #64] - stp q15, q16, [x0, #96] - stp q17, q18, [x0, #128] - stp q19, q20, [x0, #160] - stp q21, q22, [x0, #192] - stp q23, q24, [x0, #224] - ldp q9, q10, [x1] - ldp q11, q12, [x1, #32] - ldp q13, q14, [x1, #64] - ldp q15, q16, [x1, #96] - ldp q17, q18, [x1, #128] - ldp q19, q20, [x1, #160] - ldp q21, q22, [x1, #192] - ldp q23, q24, [x1, #224] - mov v25.16b, v9.16b - trn1 v9.2d, v9.2d, v10.2d - trn2 v10.2d, v25.2d, v10.2d - mov v25.16b, v9.16b - trn1 v9.4s, v9.4s, v10.4s - trn2 v10.4s, v25.4s, v10.4s - mov v25.16b, v11.16b - trn1 v11.2d, v11.2d, v12.2d - trn2 v12.2d, v25.2d, v12.2d - mov v25.16b, v11.16b - trn1 v11.4s, v11.4s, v12.4s - trn2 v12.4s, v25.4s, v12.4s - mov v25.16b, v13.16b - trn1 v13.2d, v13.2d, v14.2d - trn2 v14.2d, v25.2d, v14.2d - mov v25.16b, v13.16b - trn1 v13.4s, v13.4s, v14.4s - trn2 v14.4s, v25.4s, v14.4s - mov v25.16b, v15.16b - trn1 v15.2d, v15.2d, v16.2d - trn2 v16.2d, v25.2d, v16.2d - mov v25.16b, v15.16b - trn1 v15.4s, v15.4s, v16.4s - trn2 v16.4s, v25.4s, v16.4s - mov v25.16b, v17.16b - trn1 v17.2d, v17.2d, v18.2d - trn2 v18.2d, v25.2d, v18.2d - mov v25.16b, v17.16b - trn1 v17.4s, v17.4s, v18.4s - trn2 v18.4s, v25.4s, v18.4s - mov v25.16b, v19.16b - trn1 v19.2d, v19.2d, v20.2d - trn2 v20.2d, v25.2d, v20.2d - mov v25.16b, v19.16b - trn1 v19.4s, v19.4s, v20.4s - trn2 v20.4s, v25.4s, v20.4s - mov v25.16b, v21.16b - trn1 v21.2d, v21.2d, v22.2d - trn2 v22.2d, v25.2d, v22.2d - mov v25.16b, v21.16b - trn1 v21.4s, v21.4s, v22.4s - trn2 v22.4s, v25.4s, v22.4s - mov v25.16b, v23.16b - trn1 v23.2d, v23.2d, v24.2d - trn2 v24.2d, v25.2d, v24.2d - mov v25.16b, v23.16b - trn1 v23.4s, v23.4s, v24.4s - trn2 v24.4s, v25.4s, v24.4s - ldr q0, [x2, #128] - ldr q1, [x2, #144] - ldr q2, [x3, #128] - ldr q3, [x3, #144] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #160] - ldr q1, [x2, #176] - ldr q2, [x3, #160] - ldr q3, [x3, #176] - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #192] - ldr q1, [x2, #208] - ldr q2, [x3, #192] - ldr q3, [x3, #208] - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #224] - ldr q1, [x2, #240] - ldr q2, [x3, #224] - ldr q3, [x3, #240] - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #384] - ldr q1, [x2, #400] - ldr q2, [x3, #384] - ldr q3, [x3, #400] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.4s, v9.4s, v10.4s - trn1 v11.4s, v11.4s, v12.4s - trn2 v10.4s, v25.4s, v10.4s - trn2 v12.4s, v26.4s, v12.4s - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v10.8h, v26.8h, v0.8h - sqrdmulh v12.8h, v28.8h, v1.8h - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - ldr q0, [x2, #416] - ldr q1, [x2, #432] - ldr q2, [x3, #416] - ldr q3, [x3, #432] - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.4s, v13.4s, v14.4s - trn1 v15.4s, v15.4s, v16.4s - trn2 v14.4s, v25.4s, v14.4s - trn2 v16.4s, v26.4s, v16.4s - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v14.8h, v26.8h, v0.8h - sqrdmulh v16.8h, v28.8h, v1.8h - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - ldr q0, [x2, #448] - ldr q1, [x2, #464] - ldr q2, [x3, #448] - ldr q3, [x3, #464] - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.4s, v17.4s, v18.4s - trn1 v19.4s, v19.4s, v20.4s - trn2 v18.4s, v25.4s, v18.4s - trn2 v20.4s, v26.4s, v20.4s - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v18.8h, v26.8h, v0.8h - sqrdmulh v20.8h, v28.8h, v1.8h - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - ldr q0, [x2, #480] - ldr q1, [x2, #496] - ldr q2, [x3, #480] - ldr q3, [x3, #496] - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.4s, v21.4s, v22.4s - trn1 v23.4s, v23.4s, v24.4s - trn2 v22.4s, v25.4s, v22.4s - trn2 v24.4s, v26.4s, v24.4s - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.8h - mul v27.8h, v28.8h, v3.8h - sqrdmulh v22.8h, v26.8h, v0.8h - sqrdmulh v24.8h, v28.8h, v1.8h - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - ldr q0, [x2, #528] - ldr q2, [x3, #528] - mov v25.16b, v9.16b - mov v26.16b, v11.16b - trn1 v9.2d, v9.2d, v10.2d - trn1 v11.2d, v11.2d, v12.2d - trn2 v10.2d, v25.2d, v10.2d - trn2 v12.2d, v26.2d, v12.2d - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v2.h[0] - mul v27.8h, v28.8h, v2.h[1] - sqrdmulh v10.8h, v26.8h, v0.h[0] - sqrdmulh v12.8h, v28.8h, v0.h[1] - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - mov v25.16b, v13.16b - mov v26.16b, v15.16b - trn1 v13.2d, v13.2d, v14.2d - trn1 v15.2d, v15.2d, v16.2d - trn2 v14.2d, v25.2d, v14.2d - trn2 v16.2d, v26.2d, v16.2d - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v2.h[2] - mul v27.8h, v28.8h, v2.h[3] - sqrdmulh v14.8h, v26.8h, v0.h[2] - sqrdmulh v16.8h, v28.8h, v0.h[3] - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - mov v25.16b, v17.16b - mov v26.16b, v19.16b - trn1 v17.2d, v17.2d, v18.2d - trn1 v19.2d, v19.2d, v20.2d - trn2 v18.2d, v25.2d, v18.2d - trn2 v20.2d, v26.2d, v20.2d - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v2.h[4] - mul v27.8h, v28.8h, v2.h[5] - sqrdmulh v18.8h, v26.8h, v0.h[4] - sqrdmulh v20.8h, v28.8h, v0.h[5] - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - mov v25.16b, v21.16b - mov v26.16b, v23.16b - trn1 v21.2d, v21.2d, v22.2d - trn1 v23.2d, v23.2d, v24.2d - trn2 v22.2d, v25.2d, v22.2d - trn2 v24.2d, v26.2d, v24.2d - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v2.h[6] - mul v27.8h, v28.8h, v2.h[7] - sqrdmulh v22.8h, v26.8h, v0.h[6] - sqrdmulh v24.8h, v28.8h, v0.h[7] - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v11.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v11.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v13.8h, v8.h[2] - sqdmulh v26.8h, v15.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v13.8h, v25.8h, v8.h[0] - mls v15.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v19.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v19.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v21.8h, v8.h[2] - sqdmulh v26.8h, v23.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v21.8h, v25.8h, v8.h[0] - mls v23.8h, v26.8h, v8.h[0] - stp q9, q10, [x1] - stp q11, q12, [x1, #32] - stp q13, q14, [x1, #64] - stp q15, q16, [x1, #96] - stp q17, q18, [x1, #128] - stp q19, q20, [x1, #160] - stp q21, q22, [x1, #192] - stp q23, q24, [x1, #224] - ldr q4, [x2, #544] - ldr q5, [x2, #560] - ldr q6, [x3, #544] - ldr q7, [x3, #560] - ldr q9, [x0] - ldr q10, [x0, #32] - ldr q11, [x0, #64] - ldr q12, [x0, #96] - ldr q13, [x0, #128] - ldr q14, [x0, #160] - ldr q15, [x0, #192] - ldr q16, [x0, #224] - ldr q17, [x1] - ldr q18, [x1, #32] - ldr q19, [x1, #64] - ldr q20, [x1, #96] - ldr q21, [x1, #128] - ldr q22, [x1, #160] - ldr q23, [x1, #192] - ldr q24, [x1, #224] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v6.h[0] - mul v27.8h, v28.8h, v6.h[1] - sqrdmulh v10.8h, v26.8h, v4.h[0] - sqrdmulh v12.8h, v28.8h, v4.h[1] - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v6.h[2] - mul v27.8h, v28.8h, v6.h[3] - sqrdmulh v14.8h, v26.8h, v4.h[2] - sqrdmulh v16.8h, v28.8h, v4.h[3] - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v6.h[4] - mul v27.8h, v28.8h, v6.h[5] - sqrdmulh v18.8h, v26.8h, v4.h[4] - sqrdmulh v20.8h, v28.8h, v4.h[5] - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v6.h[6] - mul v27.8h, v28.8h, v6.h[7] - sqrdmulh v22.8h, v26.8h, v4.h[6] - sqrdmulh v24.8h, v28.8h, v4.h[7] - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v11.8h - sub v28.8h, v10.8h, v12.8h - add v9.8h, v9.8h, v11.8h - add v10.8h, v10.8h, v12.8h - mul v25.8h, v26.8h, v7.h[0] - mul v27.8h, v28.8h, v7.h[0] - sqrdmulh v11.8h, v26.8h, v5.h[0] - sqrdmulh v12.8h, v28.8h, v5.h[0] - sqrdmlsh v11.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v15.8h - sub v28.8h, v14.8h, v16.8h - add v13.8h, v13.8h, v15.8h - add v14.8h, v14.8h, v16.8h - mul v25.8h, v26.8h, v7.h[1] - mul v27.8h, v28.8h, v7.h[1] - sqrdmulh v15.8h, v26.8h, v5.h[1] - sqrdmulh v16.8h, v28.8h, v5.h[1] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v19.8h - sub v28.8h, v18.8h, v20.8h - add v17.8h, v17.8h, v19.8h - add v18.8h, v18.8h, v20.8h - mul v25.8h, v26.8h, v7.h[2] - mul v27.8h, v28.8h, v7.h[2] - sqrdmulh v19.8h, v26.8h, v5.h[2] - sqrdmulh v20.8h, v28.8h, v5.h[2] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v23.8h - sub v28.8h, v22.8h, v24.8h - add v21.8h, v21.8h, v23.8h - add v22.8h, v22.8h, v24.8h - mul v25.8h, v26.8h, v7.h[3] - mul v27.8h, v28.8h, v7.h[3] - sqrdmulh v23.8h, v26.8h, v5.h[3] - sqrdmulh v24.8h, v28.8h, v5.h[3] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v13.8h - sub v28.8h, v10.8h, v14.8h - add v9.8h, v9.8h, v13.8h - add v10.8h, v10.8h, v14.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v13.8h, v26.8h, v5.h[4] - sqrdmulh v14.8h, v28.8h, v5.h[4] - sqrdmlsh v13.8h, v25.8h, v8.h[0] - sqrdmlsh v14.8h, v27.8h, v8.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - sub v26.8h, v11.8h, v15.8h - sub v28.8h, v12.8h, v16.8h - add v11.8h, v11.8h, v15.8h - add v12.8h, v12.8h, v16.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v15.8h, v26.8h, v5.h[4] - sqrdmulh v16.8h, v28.8h, v5.h[4] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v21.8h - sub v28.8h, v18.8h, v22.8h - add v17.8h, v17.8h, v21.8h - add v18.8h, v18.8h, v22.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v21.8h, v26.8h, v5.h[5] - sqrdmulh v22.8h, v28.8h, v5.h[5] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v27.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v19.8h, v23.8h - sub v28.8h, v20.8h, v24.8h - add v19.8h, v19.8h, v23.8h - add v20.8h, v20.8h, v24.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v23.8h, v26.8h, v5.h[5] - sqrdmulh v24.8h, v28.8h, v5.h[5] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v10.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v10.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v11.8h, v8.h[2] - sqdmulh v26.8h, v12.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v11.8h, v25.8h, v8.h[0] - mls v12.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v18.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v18.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v19.8h, v8.h[2] - sqdmulh v26.8h, v20.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v19.8h, v25.8h, v8.h[0] - mls v20.8h, v26.8h, v8.h[0] - sub v26.8h, v9.8h, v17.8h - sub v28.8h, v10.8h, v18.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v17.8h, v26.8h, v5.h[6] - sqrdmulh v18.8h, v28.8h, v5.h[6] - sqrdmlsh v17.8h, v25.8h, v8.h[0] - sqrdmlsh v18.8h, v27.8h, v8.h[0] - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - sub v26.8h, v11.8h, v19.8h - sub v28.8h, v12.8h, v20.8h - add v11.8h, v11.8h, v19.8h - add v12.8h, v12.8h, v20.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v19.8h, v26.8h, v5.h[6] - sqrdmulh v20.8h, v28.8h, v5.h[6] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v13.8h, v21.8h - sub v28.8h, v14.8h, v22.8h - add v13.8h, v13.8h, v21.8h - add v14.8h, v14.8h, v22.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v21.8h, v26.8h, v5.h[6] - sqrdmulh v22.8h, v28.8h, v5.h[6] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v27.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v15.8h, v23.8h - sub v28.8h, v16.8h, v24.8h - add v15.8h, v15.8h, v23.8h - add v16.8h, v16.8h, v24.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v23.8h, v26.8h, v5.h[6] - sqrdmulh v24.8h, v28.8h, v5.h[6] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v25.8h, v9.8h, v7.h[7] - mul v26.8h, v10.8h, v7.h[7] - sqrdmulh v9.8h, v9.8h, v5.h[7] - sqrdmulh v10.8h, v10.8h, v5.h[7] - sqrdmlsh v9.8h, v25.8h, v8.h[0] - sqrdmlsh v10.8h, v26.8h, v8.h[0] - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v25.8h, v11.8h, v7.h[7] - mul v26.8h, v12.8h, v7.h[7] - sqrdmulh v11.8h, v11.8h, v5.h[7] - sqrdmulh v12.8h, v12.8h, v5.h[7] - sqrdmlsh v11.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v26.8h, v8.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v25.8h, v13.8h, v7.h[7] - mul v26.8h, v14.8h, v7.h[7] - sqrdmulh v13.8h, v13.8h, v5.h[7] - sqrdmulh v14.8h, v14.8h, v5.h[7] - sqrdmlsh v13.8h, v25.8h, v8.h[0] - sqrdmlsh v14.8h, v26.8h, v8.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v25.8h, v15.8h, v7.h[7] - mul v26.8h, v16.8h, v7.h[7] - sqrdmulh v15.8h, v15.8h, v5.h[7] - sqrdmulh v16.8h, v16.8h, v5.h[7] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v26.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - mul v25.8h, v17.8h, v7.h[7] - mul v26.8h, v18.8h, v7.h[7] - sqrdmulh v17.8h, v17.8h, v5.h[7] - sqrdmulh v18.8h, v18.8h, v5.h[7] - sqrdmlsh v17.8h, v25.8h, v8.h[0] - sqrdmlsh v18.8h, v26.8h, v8.h[0] - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - mul v25.8h, v19.8h, v7.h[7] - mul v26.8h, v20.8h, v7.h[7] - sqrdmulh v19.8h, v19.8h, v5.h[7] - sqrdmulh v20.8h, v20.8h, v5.h[7] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v26.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - mul v25.8h, v21.8h, v7.h[7] - mul v26.8h, v22.8h, v7.h[7] - sqrdmulh v21.8h, v21.8h, v5.h[7] - sqrdmulh v22.8h, v22.8h, v5.h[7] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v26.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v25.8h, v23.8h, v7.h[7] - mul v26.8h, v24.8h, v7.h[7] - sqrdmulh v23.8h, v23.8h, v5.h[7] - sqrdmulh v24.8h, v24.8h, v5.h[7] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v26.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - str q9, [x0] - str q10, [x0, #32] - str q11, [x0, #64] - str q12, [x0, #96] - str q13, [x0, #128] - str q14, [x0, #160] - str q15, [x0, #192] - str q16, [x0, #224] - str q17, [x1] - str q18, [x1, #32] - str q19, [x1, #64] - str q20, [x1, #96] - str q21, [x1, #128] - str q22, [x1, #160] - str q23, [x1, #192] - str q24, [x1, #224] - ldr q9, [x0, #16] - ldr q10, [x0, #48] - ldr q11, [x0, #80] - ldr q12, [x0, #112] - ldr q13, [x0, #144] - ldr q14, [x0, #176] - ldr q15, [x0, #208] - ldr q16, [x0, #240] - ldr q17, [x1, #16] - ldr q18, [x1, #48] - ldr q19, [x1, #80] - ldr q20, [x1, #112] - ldr q21, [x1, #144] - ldr q22, [x1, #176] - ldr q23, [x1, #208] - ldr q24, [x1, #240] - sub v26.8h, v9.8h, v10.8h - sub v28.8h, v11.8h, v12.8h - add v9.8h, v9.8h, v10.8h - add v11.8h, v11.8h, v12.8h - mul v25.8h, v26.8h, v6.h[0] - mul v27.8h, v28.8h, v6.h[1] - sqrdmulh v10.8h, v26.8h, v4.h[0] - sqrdmulh v12.8h, v28.8h, v4.h[1] - sqrdmlsh v10.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v10.8h, v10.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v14.8h - sub v28.8h, v15.8h, v16.8h - add v13.8h, v13.8h, v14.8h - add v15.8h, v15.8h, v16.8h - mul v25.8h, v26.8h, v6.h[2] - mul v27.8h, v28.8h, v6.h[3] - sqrdmulh v14.8h, v26.8h, v4.h[2] - sqrdmulh v16.8h, v28.8h, v4.h[3] - sqrdmlsh v14.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v14.8h, v14.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v18.8h - sub v28.8h, v19.8h, v20.8h - add v17.8h, v17.8h, v18.8h - add v19.8h, v19.8h, v20.8h - mul v25.8h, v26.8h, v6.h[4] - mul v27.8h, v28.8h, v6.h[5] - sqrdmulh v18.8h, v26.8h, v4.h[4] - sqrdmulh v20.8h, v28.8h, v4.h[5] - sqrdmlsh v18.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v18.8h, v18.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v22.8h - sub v28.8h, v23.8h, v24.8h - add v21.8h, v21.8h, v22.8h - add v23.8h, v23.8h, v24.8h - mul v25.8h, v26.8h, v6.h[6] - mul v27.8h, v28.8h, v6.h[7] - sqrdmulh v22.8h, v26.8h, v4.h[6] - sqrdmulh v24.8h, v28.8h, v4.h[7] - sqrdmlsh v22.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v22.8h, v22.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v11.8h - sub v28.8h, v10.8h, v12.8h - add v9.8h, v9.8h, v11.8h - add v10.8h, v10.8h, v12.8h - mul v25.8h, v26.8h, v7.h[0] - mul v27.8h, v28.8h, v7.h[0] - sqrdmulh v11.8h, v26.8h, v5.h[0] - sqrdmulh v12.8h, v28.8h, v5.h[0] - sqrdmlsh v11.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v27.8h, v8.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - sub v26.8h, v13.8h, v15.8h - sub v28.8h, v14.8h, v16.8h - add v13.8h, v13.8h, v15.8h - add v14.8h, v14.8h, v16.8h - mul v25.8h, v26.8h, v7.h[1] - mul v27.8h, v28.8h, v7.h[1] - sqrdmulh v15.8h, v26.8h, v5.h[1] - sqrdmulh v16.8h, v28.8h, v5.h[1] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v19.8h - sub v28.8h, v18.8h, v20.8h - add v17.8h, v17.8h, v19.8h - add v18.8h, v18.8h, v20.8h - mul v25.8h, v26.8h, v7.h[2] - mul v27.8h, v28.8h, v7.h[2] - sqrdmulh v19.8h, v26.8h, v5.h[2] - sqrdmulh v20.8h, v28.8h, v5.h[2] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v21.8h, v23.8h - sub v28.8h, v22.8h, v24.8h - add v21.8h, v21.8h, v23.8h - add v22.8h, v22.8h, v24.8h - mul v25.8h, v26.8h, v7.h[3] - mul v27.8h, v28.8h, v7.h[3] - sqrdmulh v23.8h, v26.8h, v5.h[3] - sqrdmulh v24.8h, v28.8h, v5.h[3] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sub v26.8h, v9.8h, v13.8h - sub v28.8h, v10.8h, v14.8h - add v9.8h, v9.8h, v13.8h - add v10.8h, v10.8h, v14.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v13.8h, v26.8h, v5.h[4] - sqrdmulh v14.8h, v28.8h, v5.h[4] - sqrdmlsh v13.8h, v25.8h, v8.h[0] - sqrdmlsh v14.8h, v27.8h, v8.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - sub v26.8h, v11.8h, v15.8h - sub v28.8h, v12.8h, v16.8h - add v11.8h, v11.8h, v15.8h - add v12.8h, v12.8h, v16.8h - mul v25.8h, v26.8h, v7.h[4] - mul v27.8h, v28.8h, v7.h[4] - sqrdmulh v15.8h, v26.8h, v5.h[4] - sqrdmulh v16.8h, v28.8h, v5.h[4] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v27.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - sub v26.8h, v17.8h, v21.8h - sub v28.8h, v18.8h, v22.8h - add v17.8h, v17.8h, v21.8h - add v18.8h, v18.8h, v22.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v21.8h, v26.8h, v5.h[5] - sqrdmulh v22.8h, v28.8h, v5.h[5] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v27.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v19.8h, v23.8h - sub v28.8h, v20.8h, v24.8h - add v19.8h, v19.8h, v23.8h - add v20.8h, v20.8h, v24.8h - mul v25.8h, v26.8h, v7.h[5] - mul v27.8h, v28.8h, v7.h[5] - sqrdmulh v23.8h, v26.8h, v5.h[5] - sqrdmulh v24.8h, v28.8h, v5.h[5] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - sqdmulh v25.8h, v9.8h, v8.h[2] - sqdmulh v26.8h, v10.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v9.8h, v25.8h, v8.h[0] - mls v10.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v11.8h, v8.h[2] - sqdmulh v26.8h, v12.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v11.8h, v25.8h, v8.h[0] - mls v12.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v17.8h, v8.h[2] - sqdmulh v26.8h, v18.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v17.8h, v25.8h, v8.h[0] - mls v18.8h, v26.8h, v8.h[0] - sqdmulh v25.8h, v19.8h, v8.h[2] - sqdmulh v26.8h, v20.8h, v8.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v19.8h, v25.8h, v8.h[0] - mls v20.8h, v26.8h, v8.h[0] - sub v26.8h, v9.8h, v17.8h - sub v28.8h, v10.8h, v18.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v17.8h, v26.8h, v5.h[6] - sqrdmulh v18.8h, v28.8h, v5.h[6] - sqrdmlsh v17.8h, v25.8h, v8.h[0] - sqrdmlsh v18.8h, v27.8h, v8.h[0] - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - sub v26.8h, v11.8h, v19.8h - sub v28.8h, v12.8h, v20.8h - add v11.8h, v11.8h, v19.8h - add v12.8h, v12.8h, v20.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v19.8h, v26.8h, v5.h[6] - sqrdmulh v20.8h, v28.8h, v5.h[6] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v27.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - sub v26.8h, v13.8h, v21.8h - sub v28.8h, v14.8h, v22.8h - add v13.8h, v13.8h, v21.8h - add v14.8h, v14.8h, v22.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v21.8h, v26.8h, v5.h[6] - sqrdmulh v22.8h, v28.8h, v5.h[6] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v27.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - sub v26.8h, v15.8h, v23.8h - sub v28.8h, v16.8h, v24.8h - add v15.8h, v15.8h, v23.8h - add v16.8h, v16.8h, v24.8h - mul v25.8h, v26.8h, v7.h[6] - mul v27.8h, v28.8h, v7.h[6] - sqrdmulh v23.8h, v26.8h, v5.h[6] - sqrdmulh v24.8h, v28.8h, v5.h[6] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v27.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - mul v25.8h, v9.8h, v7.h[7] - mul v26.8h, v10.8h, v7.h[7] - sqrdmulh v9.8h, v9.8h, v5.h[7] - sqrdmulh v10.8h, v10.8h, v5.h[7] - sqrdmlsh v9.8h, v25.8h, v8.h[0] - sqrdmlsh v10.8h, v26.8h, v8.h[0] - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v25.8h, v11.8h, v7.h[7] - mul v26.8h, v12.8h, v7.h[7] - sqrdmulh v11.8h, v11.8h, v5.h[7] - sqrdmulh v12.8h, v12.8h, v5.h[7] - sqrdmlsh v11.8h, v25.8h, v8.h[0] - sqrdmlsh v12.8h, v26.8h, v8.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v25.8h, v13.8h, v7.h[7] - mul v26.8h, v14.8h, v7.h[7] - sqrdmulh v13.8h, v13.8h, v5.h[7] - sqrdmulh v14.8h, v14.8h, v5.h[7] - sqrdmlsh v13.8h, v25.8h, v8.h[0] - sqrdmlsh v14.8h, v26.8h, v8.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v25.8h, v15.8h, v7.h[7] - mul v26.8h, v16.8h, v7.h[7] - sqrdmulh v15.8h, v15.8h, v5.h[7] - sqrdmulh v16.8h, v16.8h, v5.h[7] - sqrdmlsh v15.8h, v25.8h, v8.h[0] - sqrdmlsh v16.8h, v26.8h, v8.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - mul v25.8h, v17.8h, v7.h[7] - mul v26.8h, v18.8h, v7.h[7] - sqrdmulh v17.8h, v17.8h, v5.h[7] - sqrdmulh v18.8h, v18.8h, v5.h[7] - sqrdmlsh v17.8h, v25.8h, v8.h[0] - sqrdmlsh v18.8h, v26.8h, v8.h[0] - sshr v17.8h, v17.8h, #1 - sshr v18.8h, v18.8h, #1 - mul v25.8h, v19.8h, v7.h[7] - mul v26.8h, v20.8h, v7.h[7] - sqrdmulh v19.8h, v19.8h, v5.h[7] - sqrdmulh v20.8h, v20.8h, v5.h[7] - sqrdmlsh v19.8h, v25.8h, v8.h[0] - sqrdmlsh v20.8h, v26.8h, v8.h[0] - sshr v19.8h, v19.8h, #1 - sshr v20.8h, v20.8h, #1 - mul v25.8h, v21.8h, v7.h[7] - mul v26.8h, v22.8h, v7.h[7] - sqrdmulh v21.8h, v21.8h, v5.h[7] - sqrdmulh v22.8h, v22.8h, v5.h[7] - sqrdmlsh v21.8h, v25.8h, v8.h[0] - sqrdmlsh v22.8h, v26.8h, v8.h[0] - sshr v21.8h, v21.8h, #1 - sshr v22.8h, v22.8h, #1 - mul v25.8h, v23.8h, v7.h[7] - mul v26.8h, v24.8h, v7.h[7] - sqrdmulh v23.8h, v23.8h, v5.h[7] - sqrdmulh v24.8h, v24.8h, v5.h[7] - sqrdmlsh v23.8h, v25.8h, v8.h[0] - sqrdmlsh v24.8h, v26.8h, v8.h[0] - sshr v23.8h, v23.8h, #1 - sshr v24.8h, v24.8h, #1 - str q9, [x0, #16] - str q10, [x0, #48] - str q11, [x0, #80] - str q12, [x0, #112] - str q13, [x0, #144] - str q14, [x0, #176] - str q15, [x0, #208] - str q16, [x0, #240] - str q17, [x1, #16] - str q18, [x1, #48] - str q19, [x1, #80] - str q20, [x1, #112] - str q21, [x1, #144] - str q22, [x1, #176] - str q23, [x1, #208] - str q24, [x1, #240] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q8, [x4] + ldp Q9, Q10, [x0] + ldp Q11, Q12, [x0, #32] + ldp Q13, Q14, [x0, #64] + ldp Q15, Q16, [x0, #96] + ldp Q17, Q18, [x0, #128] + ldp Q19, Q20, [x0, #160] + ldp Q21, Q22, [x0, #192] + ldp Q23, Q24, [x0, #224] + mov V25.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V25.2D, V10.2D + mov V25.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V25.4S, V10.4S + mov V25.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V25.2D, V12.2D + mov V25.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V25.4S, V12.4S + mov V25.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V25.2D, V14.2D + mov V25.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V25.4S, V14.4S + mov V25.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V25.2D, V16.2D + mov V25.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V25.4S, V16.4S + mov V25.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V25.2D, V18.2D + mov V25.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V25.4S, V18.4S + mov V25.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V25.2D, V20.2D + mov V25.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V25.4S, V20.4S + mov V25.16B, V21.16B + trn1 V21.2D, V21.2D, V22.2D + trn2 V22.2D, V25.2D, V22.2D + mov V25.16B, V21.16B + trn1 V21.4S, V21.4S, V22.4S + trn2 V22.4S, V25.4S, V22.4S + mov V25.16B, V23.16B + trn1 V23.2D, V23.2D, V24.2D + trn2 V24.2D, V25.2D, V24.2D + mov V25.16B, V23.16B + trn1 V23.4S, V23.4S, V24.4S + trn2 V24.4S, V25.4S, V24.4S + ldr Q0, [x2] + ldr Q1, [x2, #16] + ldr Q2, [x3] + ldr Q3, [x3, #16] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #32] + ldr Q1, [x2, #48] + ldr Q2, [x3, #32] + ldr Q3, [x3, #48] + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #64] + ldr Q1, [x2, #80] + ldr Q2, [x3, #64] + ldr Q3, [x3, #80] + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #96] + ldr Q1, [x2, #112] + ldr Q2, [x3, #96] + ldr Q3, [x3, #112] + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #256] + ldr Q1, [x2, #272] + ldr Q2, [x3, #256] + ldr Q3, [x3, #272] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V25.4S, V10.4S + trn2 V12.4S, V26.4S, V12.4S + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #288] + ldr Q1, [x2, #304] + ldr Q2, [x3, #288] + ldr Q3, [x3, #304] + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V25.4S, V14.4S + trn2 V16.4S, V26.4S, V16.4S + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #320] + ldr Q1, [x2, #336] + ldr Q2, [x3, #320] + ldr Q3, [x3, #336] + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V25.4S, V18.4S + trn2 V20.4S, V26.4S, V20.4S + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #352] + ldr Q1, [x2, #368] + ldr Q2, [x3, #352] + ldr Q3, [x3, #368] + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.4S, V21.4S, V22.4S + trn1 V23.4S, V23.4S, V24.4S + trn2 V22.4S, V25.4S, V22.4S + trn2 V24.4S, V26.4S, V24.4S + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #512] + ldr Q2, [x3, #512] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V25.2D, V10.2D + trn2 V12.2D, V26.2D, V12.2D + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.H[0] + mul V27.8H, V28.8H, V2.H[1] + sqrdmulh V10.8H, V26.8H, V0.H[0] + sqrdmulh V12.8H, V28.8H, V0.H[1] + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V25.2D, V14.2D + trn2 V16.2D, V26.2D, V16.2D + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.H[2] + mul V27.8H, V28.8H, V2.H[3] + sqrdmulh V14.8H, V26.8H, V0.H[2] + sqrdmulh V16.8H, V28.8H, V0.H[3] + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V25.2D, V18.2D + trn2 V20.2D, V26.2D, V20.2D + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.H[4] + mul V27.8H, V28.8H, V2.H[5] + sqrdmulh V18.8H, V26.8H, V0.H[4] + sqrdmulh V20.8H, V28.8H, V0.H[5] + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.2D, V21.2D, V22.2D + trn1 V23.2D, V23.2D, V24.2D + trn2 V22.2D, V25.2D, V22.2D + trn2 V24.2D, V26.2D, V24.2D + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.H[6] + mul V27.8H, V28.8H, V2.H[7] + sqrdmulh V22.8H, V26.8H, V0.H[6] + sqrdmulh V24.8H, V28.8H, V0.H[7] + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V11.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V11.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V13.8H, V8.H[2] + sqdmulh V26.8H, V15.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V13.8H, V25.8H, V8.H[0] + mls V15.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V19.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V19.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V21.8H, V8.H[2] + sqdmulh V26.8H, V23.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V21.8H, V25.8H, V8.H[0] + mls V23.8H, V26.8H, V8.H[0] + stp Q9, Q10, [x0] + stp Q11, Q12, [x0, #32] + stp Q13, Q14, [x0, #64] + stp Q15, Q16, [x0, #96] + stp Q17, Q18, [x0, #128] + stp Q19, Q20, [x0, #160] + stp Q21, Q22, [x0, #192] + stp Q23, Q24, [x0, #224] + ldp Q9, Q10, [x1] + ldp Q11, Q12, [x1, #32] + ldp Q13, Q14, [x1, #64] + ldp Q15, Q16, [x1, #96] + ldp Q17, Q18, [x1, #128] + ldp Q19, Q20, [x1, #160] + ldp Q21, Q22, [x1, #192] + ldp Q23, Q24, [x1, #224] + mov V25.16B, V9.16B + trn1 V9.2D, V9.2D, V10.2D + trn2 V10.2D, V25.2D, V10.2D + mov V25.16B, V9.16B + trn1 V9.4S, V9.4S, V10.4S + trn2 V10.4S, V25.4S, V10.4S + mov V25.16B, V11.16B + trn1 V11.2D, V11.2D, V12.2D + trn2 V12.2D, V25.2D, V12.2D + mov V25.16B, V11.16B + trn1 V11.4S, V11.4S, V12.4S + trn2 V12.4S, V25.4S, V12.4S + mov V25.16B, V13.16B + trn1 V13.2D, V13.2D, V14.2D + trn2 V14.2D, V25.2D, V14.2D + mov V25.16B, V13.16B + trn1 V13.4S, V13.4S, V14.4S + trn2 V14.4S, V25.4S, V14.4S + mov V25.16B, V15.16B + trn1 V15.2D, V15.2D, V16.2D + trn2 V16.2D, V25.2D, V16.2D + mov V25.16B, V15.16B + trn1 V15.4S, V15.4S, V16.4S + trn2 V16.4S, V25.4S, V16.4S + mov V25.16B, V17.16B + trn1 V17.2D, V17.2D, V18.2D + trn2 V18.2D, V25.2D, V18.2D + mov V25.16B, V17.16B + trn1 V17.4S, V17.4S, V18.4S + trn2 V18.4S, V25.4S, V18.4S + mov V25.16B, V19.16B + trn1 V19.2D, V19.2D, V20.2D + trn2 V20.2D, V25.2D, V20.2D + mov V25.16B, V19.16B + trn1 V19.4S, V19.4S, V20.4S + trn2 V20.4S, V25.4S, V20.4S + mov V25.16B, V21.16B + trn1 V21.2D, V21.2D, V22.2D + trn2 V22.2D, V25.2D, V22.2D + mov V25.16B, V21.16B + trn1 V21.4S, V21.4S, V22.4S + trn2 V22.4S, V25.4S, V22.4S + mov V25.16B, V23.16B + trn1 V23.2D, V23.2D, V24.2D + trn2 V24.2D, V25.2D, V24.2D + mov V25.16B, V23.16B + trn1 V23.4S, V23.4S, V24.4S + trn2 V24.4S, V25.4S, V24.4S + ldr Q0, [x2, #128] + ldr Q1, [x2, #144] + ldr Q2, [x3, #128] + ldr Q3, [x3, #144] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #160] + ldr Q1, [x2, #176] + ldr Q2, [x3, #160] + ldr Q3, [x3, #176] + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #192] + ldr Q1, [x2, #208] + ldr Q2, [x3, #192] + ldr Q3, [x3, #208] + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #224] + ldr Q1, [x2, #240] + ldr Q2, [x3, #224] + ldr Q3, [x3, #240] + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #384] + ldr Q1, [x2, #400] + ldr Q2, [x3, #384] + ldr Q3, [x3, #400] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V25.4S, V10.4S + trn2 V12.4S, V26.4S, V12.4S + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V10.8H, V26.8H, V0.8H + sqrdmulh V12.8H, V28.8H, V1.8H + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + ldr Q0, [x2, #416] + ldr Q1, [x2, #432] + ldr Q2, [x3, #416] + ldr Q3, [x3, #432] + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V25.4S, V14.4S + trn2 V16.4S, V26.4S, V16.4S + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V14.8H, V26.8H, V0.8H + sqrdmulh V16.8H, V28.8H, V1.8H + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + ldr Q0, [x2, #448] + ldr Q1, [x2, #464] + ldr Q2, [x3, #448] + ldr Q3, [x3, #464] + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V25.4S, V18.4S + trn2 V20.4S, V26.4S, V20.4S + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V18.8H, V26.8H, V0.8H + sqrdmulh V20.8H, V28.8H, V1.8H + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + ldr Q0, [x2, #480] + ldr Q1, [x2, #496] + ldr Q2, [x3, #480] + ldr Q3, [x3, #496] + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.4S, V21.4S, V22.4S + trn1 V23.4S, V23.4S, V24.4S + trn2 V22.4S, V25.4S, V22.4S + trn2 V24.4S, V26.4S, V24.4S + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.8H + mul V27.8H, V28.8H, V3.8H + sqrdmulh V22.8H, V26.8H, V0.8H + sqrdmulh V24.8H, V28.8H, V1.8H + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + ldr Q0, [x2, #528] + ldr Q2, [x3, #528] + mov V25.16B, V9.16B + mov V26.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V25.2D, V10.2D + trn2 V12.2D, V26.2D, V12.2D + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V2.H[0] + mul V27.8H, V28.8H, V2.H[1] + sqrdmulh V10.8H, V26.8H, V0.H[0] + sqrdmulh V12.8H, V28.8H, V0.H[1] + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + mov V25.16B, V13.16B + mov V26.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V25.2D, V14.2D + trn2 V16.2D, V26.2D, V16.2D + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V2.H[2] + mul V27.8H, V28.8H, V2.H[3] + sqrdmulh V14.8H, V26.8H, V0.H[2] + sqrdmulh V16.8H, V28.8H, V0.H[3] + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + mov V25.16B, V17.16B + mov V26.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V25.2D, V18.2D + trn2 V20.2D, V26.2D, V20.2D + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V2.H[4] + mul V27.8H, V28.8H, V2.H[5] + sqrdmulh V18.8H, V26.8H, V0.H[4] + sqrdmulh V20.8H, V28.8H, V0.H[5] + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + mov V25.16B, V21.16B + mov V26.16B, V23.16B + trn1 V21.2D, V21.2D, V22.2D + trn1 V23.2D, V23.2D, V24.2D + trn2 V22.2D, V25.2D, V22.2D + trn2 V24.2D, V26.2D, V24.2D + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V2.H[6] + mul V27.8H, V28.8H, V2.H[7] + sqrdmulh V22.8H, V26.8H, V0.H[6] + sqrdmulh V24.8H, V28.8H, V0.H[7] + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V11.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V11.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V13.8H, V8.H[2] + sqdmulh V26.8H, V15.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V13.8H, V25.8H, V8.H[0] + mls V15.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V19.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V19.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V21.8H, V8.H[2] + sqdmulh V26.8H, V23.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V21.8H, V25.8H, V8.H[0] + mls V23.8H, V26.8H, V8.H[0] + stp Q9, Q10, [x1] + stp Q11, Q12, [x1, #32] + stp Q13, Q14, [x1, #64] + stp Q15, Q16, [x1, #96] + stp Q17, Q18, [x1, #128] + stp Q19, Q20, [x1, #160] + stp Q21, Q22, [x1, #192] + stp Q23, Q24, [x1, #224] + ldr Q4, [x2, #544] + ldr Q5, [x2, #560] + ldr Q6, [x3, #544] + ldr Q7, [x3, #560] + ldr Q9, [x0] + ldr Q10, [x0, #32] + ldr Q11, [x0, #64] + ldr Q12, [x0, #96] + ldr Q13, [x0, #128] + ldr Q14, [x0, #160] + ldr Q15, [x0, #192] + ldr Q16, [x0, #224] + ldr Q17, [x1] + ldr Q18, [x1, #32] + ldr Q19, [x1, #64] + ldr Q20, [x1, #96] + ldr Q21, [x1, #128] + ldr Q22, [x1, #160] + ldr Q23, [x1, #192] + ldr Q24, [x1, #224] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V6.H[0] + mul V27.8H, V28.8H, V6.H[1] + sqrdmulh V10.8H, V26.8H, V4.H[0] + sqrdmulh V12.8H, V28.8H, V4.H[1] + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V6.H[2] + mul V27.8H, V28.8H, V6.H[3] + sqrdmulh V14.8H, V26.8H, V4.H[2] + sqrdmulh V16.8H, V28.8H, V4.H[3] + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V6.H[4] + mul V27.8H, V28.8H, V6.H[5] + sqrdmulh V18.8H, V26.8H, V4.H[4] + sqrdmulh V20.8H, V28.8H, V4.H[5] + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V6.H[6] + mul V27.8H, V28.8H, V6.H[7] + sqrdmulh V22.8H, V26.8H, V4.H[6] + sqrdmulh V24.8H, V28.8H, V4.H[7] + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V11.8H + sub V28.8H, V10.8H, V12.8H + add V9.8H, V9.8H, V11.8H + add V10.8H, V10.8H, V12.8H + mul V25.8H, V26.8H, V7.H[0] + mul V27.8H, V28.8H, V7.H[0] + sqrdmulh V11.8H, V26.8H, V5.H[0] + sqrdmulh V12.8H, V28.8H, V5.H[0] + sqrdmlsh V11.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V15.8H + sub V28.8H, V14.8H, V16.8H + add V13.8H, V13.8H, V15.8H + add V14.8H, V14.8H, V16.8H + mul V25.8H, V26.8H, V7.H[1] + mul V27.8H, V28.8H, V7.H[1] + sqrdmulh V15.8H, V26.8H, V5.H[1] + sqrdmulh V16.8H, V28.8H, V5.H[1] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V19.8H + sub V28.8H, V18.8H, V20.8H + add V17.8H, V17.8H, V19.8H + add V18.8H, V18.8H, V20.8H + mul V25.8H, V26.8H, V7.H[2] + mul V27.8H, V28.8H, V7.H[2] + sqrdmulh V19.8H, V26.8H, V5.H[2] + sqrdmulh V20.8H, V28.8H, V5.H[2] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V23.8H + sub V28.8H, V22.8H, V24.8H + add V21.8H, V21.8H, V23.8H + add V22.8H, V22.8H, V24.8H + mul V25.8H, V26.8H, V7.H[3] + mul V27.8H, V28.8H, V7.H[3] + sqrdmulh V23.8H, V26.8H, V5.H[3] + sqrdmulh V24.8H, V28.8H, V5.H[3] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V13.8H + sub V28.8H, V10.8H, V14.8H + add V9.8H, V9.8H, V13.8H + add V10.8H, V10.8H, V14.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V13.8H, V26.8H, V5.H[4] + sqrdmulh V14.8H, V28.8H, V5.H[4] + sqrdmlsh V13.8H, V25.8H, V8.H[0] + sqrdmlsh V14.8H, V27.8H, V8.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + sub V26.8H, V11.8H, V15.8H + sub V28.8H, V12.8H, V16.8H + add V11.8H, V11.8H, V15.8H + add V12.8H, V12.8H, V16.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V15.8H, V26.8H, V5.H[4] + sqrdmulh V16.8H, V28.8H, V5.H[4] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V21.8H + sub V28.8H, V18.8H, V22.8H + add V17.8H, V17.8H, V21.8H + add V18.8H, V18.8H, V22.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V21.8H, V26.8H, V5.H[5] + sqrdmulh V22.8H, V28.8H, V5.H[5] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V27.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V19.8H, V23.8H + sub V28.8H, V20.8H, V24.8H + add V19.8H, V19.8H, V23.8H + add V20.8H, V20.8H, V24.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V23.8H, V26.8H, V5.H[5] + sqrdmulh V24.8H, V28.8H, V5.H[5] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V10.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V10.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V11.8H, V8.H[2] + sqdmulh V26.8H, V12.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V11.8H, V25.8H, V8.H[0] + mls V12.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V18.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V18.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V19.8H, V8.H[2] + sqdmulh V26.8H, V20.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V19.8H, V25.8H, V8.H[0] + mls V20.8H, V26.8H, V8.H[0] + sub V26.8H, V9.8H, V17.8H + sub V28.8H, V10.8H, V18.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V17.8H, V26.8H, V5.H[6] + sqrdmulh V18.8H, V28.8H, V5.H[6] + sqrdmlsh V17.8H, V25.8H, V8.H[0] + sqrdmlsh V18.8H, V27.8H, V8.H[0] + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + sub V26.8H, V11.8H, V19.8H + sub V28.8H, V12.8H, V20.8H + add V11.8H, V11.8H, V19.8H + add V12.8H, V12.8H, V20.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V19.8H, V26.8H, V5.H[6] + sqrdmulh V20.8H, V28.8H, V5.H[6] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V13.8H, V21.8H + sub V28.8H, V14.8H, V22.8H + add V13.8H, V13.8H, V21.8H + add V14.8H, V14.8H, V22.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V21.8H, V26.8H, V5.H[6] + sqrdmulh V22.8H, V28.8H, V5.H[6] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V27.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V15.8H, V23.8H + sub V28.8H, V16.8H, V24.8H + add V15.8H, V15.8H, V23.8H + add V16.8H, V16.8H, V24.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V23.8H, V26.8H, V5.H[6] + sqrdmulh V24.8H, V28.8H, V5.H[6] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V25.8H, V9.8H, V7.H[7] + mul V26.8H, V10.8H, V7.H[7] + sqrdmulh V9.8H, V9.8H, V5.H[7] + sqrdmulh V10.8H, V10.8H, V5.H[7] + sqrdmlsh V9.8H, V25.8H, V8.H[0] + sqrdmlsh V10.8H, V26.8H, V8.H[0] + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V25.8H, V11.8H, V7.H[7] + mul V26.8H, V12.8H, V7.H[7] + sqrdmulh V11.8H, V11.8H, V5.H[7] + sqrdmulh V12.8H, V12.8H, V5.H[7] + sqrdmlsh V11.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V26.8H, V8.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V25.8H, V13.8H, V7.H[7] + mul V26.8H, V14.8H, V7.H[7] + sqrdmulh V13.8H, V13.8H, V5.H[7] + sqrdmulh V14.8H, V14.8H, V5.H[7] + sqrdmlsh V13.8H, V25.8H, V8.H[0] + sqrdmlsh V14.8H, V26.8H, V8.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V25.8H, V15.8H, V7.H[7] + mul V26.8H, V16.8H, V7.H[7] + sqrdmulh V15.8H, V15.8H, V5.H[7] + sqrdmulh V16.8H, V16.8H, V5.H[7] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V26.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + mul V25.8H, V17.8H, V7.H[7] + mul V26.8H, V18.8H, V7.H[7] + sqrdmulh V17.8H, V17.8H, V5.H[7] + sqrdmulh V18.8H, V18.8H, V5.H[7] + sqrdmlsh V17.8H, V25.8H, V8.H[0] + sqrdmlsh V18.8H, V26.8H, V8.H[0] + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + mul V25.8H, V19.8H, V7.H[7] + mul V26.8H, V20.8H, V7.H[7] + sqrdmulh V19.8H, V19.8H, V5.H[7] + sqrdmulh V20.8H, V20.8H, V5.H[7] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V26.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + mul V25.8H, V21.8H, V7.H[7] + mul V26.8H, V22.8H, V7.H[7] + sqrdmulh V21.8H, V21.8H, V5.H[7] + sqrdmulh V22.8H, V22.8H, V5.H[7] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V26.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V25.8H, V23.8H, V7.H[7] + mul V26.8H, V24.8H, V7.H[7] + sqrdmulh V23.8H, V23.8H, V5.H[7] + sqrdmulh V24.8H, V24.8H, V5.H[7] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V26.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + str Q9, [x0] + str Q10, [x0, #32] + str Q11, [x0, #64] + str Q12, [x0, #96] + str Q13, [x0, #128] + str Q14, [x0, #160] + str Q15, [x0, #192] + str Q16, [x0, #224] + str Q17, [x1] + str Q18, [x1, #32] + str Q19, [x1, #64] + str Q20, [x1, #96] + str Q21, [x1, #128] + str Q22, [x1, #160] + str Q23, [x1, #192] + str Q24, [x1, #224] + ldr Q9, [x0, #16] + ldr Q10, [x0, #48] + ldr Q11, [x0, #80] + ldr Q12, [x0, #112] + ldr Q13, [x0, #144] + ldr Q14, [x0, #176] + ldr Q15, [x0, #208] + ldr Q16, [x0, #240] + ldr Q17, [x1, #16] + ldr Q18, [x1, #48] + ldr Q19, [x1, #80] + ldr Q20, [x1, #112] + ldr Q21, [x1, #144] + ldr Q22, [x1, #176] + ldr Q23, [x1, #208] + ldr Q24, [x1, #240] + sub V26.8H, V9.8H, V10.8H + sub V28.8H, V11.8H, V12.8H + add V9.8H, V9.8H, V10.8H + add V11.8H, V11.8H, V12.8H + mul V25.8H, V26.8H, V6.H[0] + mul V27.8H, V28.8H, V6.H[1] + sqrdmulh V10.8H, V26.8H, V4.H[0] + sqrdmulh V12.8H, V28.8H, V4.H[1] + sqrdmlsh V10.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V10.8H, V10.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V14.8H + sub V28.8H, V15.8H, V16.8H + add V13.8H, V13.8H, V14.8H + add V15.8H, V15.8H, V16.8H + mul V25.8H, V26.8H, V6.H[2] + mul V27.8H, V28.8H, V6.H[3] + sqrdmulh V14.8H, V26.8H, V4.H[2] + sqrdmulh V16.8H, V28.8H, V4.H[3] + sqrdmlsh V14.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V14.8H, V14.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V18.8H + sub V28.8H, V19.8H, V20.8H + add V17.8H, V17.8H, V18.8H + add V19.8H, V19.8H, V20.8H + mul V25.8H, V26.8H, V6.H[4] + mul V27.8H, V28.8H, V6.H[5] + sqrdmulh V18.8H, V26.8H, V4.H[4] + sqrdmulh V20.8H, V28.8H, V4.H[5] + sqrdmlsh V18.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V18.8H, V18.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V22.8H + sub V28.8H, V23.8H, V24.8H + add V21.8H, V21.8H, V22.8H + add V23.8H, V23.8H, V24.8H + mul V25.8H, V26.8H, V6.H[6] + mul V27.8H, V28.8H, V6.H[7] + sqrdmulh V22.8H, V26.8H, V4.H[6] + sqrdmulh V24.8H, V28.8H, V4.H[7] + sqrdmlsh V22.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V22.8H, V22.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V11.8H + sub V28.8H, V10.8H, V12.8H + add V9.8H, V9.8H, V11.8H + add V10.8H, V10.8H, V12.8H + mul V25.8H, V26.8H, V7.H[0] + mul V27.8H, V28.8H, V7.H[0] + sqrdmulh V11.8H, V26.8H, V5.H[0] + sqrdmulh V12.8H, V28.8H, V5.H[0] + sqrdmlsh V11.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V27.8H, V8.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + sub V26.8H, V13.8H, V15.8H + sub V28.8H, V14.8H, V16.8H + add V13.8H, V13.8H, V15.8H + add V14.8H, V14.8H, V16.8H + mul V25.8H, V26.8H, V7.H[1] + mul V27.8H, V28.8H, V7.H[1] + sqrdmulh V15.8H, V26.8H, V5.H[1] + sqrdmulh V16.8H, V28.8H, V5.H[1] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V19.8H + sub V28.8H, V18.8H, V20.8H + add V17.8H, V17.8H, V19.8H + add V18.8H, V18.8H, V20.8H + mul V25.8H, V26.8H, V7.H[2] + mul V27.8H, V28.8H, V7.H[2] + sqrdmulh V19.8H, V26.8H, V5.H[2] + sqrdmulh V20.8H, V28.8H, V5.H[2] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V21.8H, V23.8H + sub V28.8H, V22.8H, V24.8H + add V21.8H, V21.8H, V23.8H + add V22.8H, V22.8H, V24.8H + mul V25.8H, V26.8H, V7.H[3] + mul V27.8H, V28.8H, V7.H[3] + sqrdmulh V23.8H, V26.8H, V5.H[3] + sqrdmulh V24.8H, V28.8H, V5.H[3] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sub V26.8H, V9.8H, V13.8H + sub V28.8H, V10.8H, V14.8H + add V9.8H, V9.8H, V13.8H + add V10.8H, V10.8H, V14.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V13.8H, V26.8H, V5.H[4] + sqrdmulh V14.8H, V28.8H, V5.H[4] + sqrdmlsh V13.8H, V25.8H, V8.H[0] + sqrdmlsh V14.8H, V27.8H, V8.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + sub V26.8H, V11.8H, V15.8H + sub V28.8H, V12.8H, V16.8H + add V11.8H, V11.8H, V15.8H + add V12.8H, V12.8H, V16.8H + mul V25.8H, V26.8H, V7.H[4] + mul V27.8H, V28.8H, V7.H[4] + sqrdmulh V15.8H, V26.8H, V5.H[4] + sqrdmulh V16.8H, V28.8H, V5.H[4] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V27.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + sub V26.8H, V17.8H, V21.8H + sub V28.8H, V18.8H, V22.8H + add V17.8H, V17.8H, V21.8H + add V18.8H, V18.8H, V22.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V21.8H, V26.8H, V5.H[5] + sqrdmulh V22.8H, V28.8H, V5.H[5] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V27.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V19.8H, V23.8H + sub V28.8H, V20.8H, V24.8H + add V19.8H, V19.8H, V23.8H + add V20.8H, V20.8H, V24.8H + mul V25.8H, V26.8H, V7.H[5] + mul V27.8H, V28.8H, V7.H[5] + sqrdmulh V23.8H, V26.8H, V5.H[5] + sqrdmulh V24.8H, V28.8H, V5.H[5] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + sqdmulh V25.8H, V9.8H, V8.H[2] + sqdmulh V26.8H, V10.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V9.8H, V25.8H, V8.H[0] + mls V10.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V11.8H, V8.H[2] + sqdmulh V26.8H, V12.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V11.8H, V25.8H, V8.H[0] + mls V12.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V17.8H, V8.H[2] + sqdmulh V26.8H, V18.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V17.8H, V25.8H, V8.H[0] + mls V18.8H, V26.8H, V8.H[0] + sqdmulh V25.8H, V19.8H, V8.H[2] + sqdmulh V26.8H, V20.8H, V8.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V19.8H, V25.8H, V8.H[0] + mls V20.8H, V26.8H, V8.H[0] + sub V26.8H, V9.8H, V17.8H + sub V28.8H, V10.8H, V18.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V17.8H, V26.8H, V5.H[6] + sqrdmulh V18.8H, V28.8H, V5.H[6] + sqrdmlsh V17.8H, V25.8H, V8.H[0] + sqrdmlsh V18.8H, V27.8H, V8.H[0] + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + sub V26.8H, V11.8H, V19.8H + sub V28.8H, V12.8H, V20.8H + add V11.8H, V11.8H, V19.8H + add V12.8H, V12.8H, V20.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V19.8H, V26.8H, V5.H[6] + sqrdmulh V20.8H, V28.8H, V5.H[6] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V27.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + sub V26.8H, V13.8H, V21.8H + sub V28.8H, V14.8H, V22.8H + add V13.8H, V13.8H, V21.8H + add V14.8H, V14.8H, V22.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V21.8H, V26.8H, V5.H[6] + sqrdmulh V22.8H, V28.8H, V5.H[6] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V27.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + sub V26.8H, V15.8H, V23.8H + sub V28.8H, V16.8H, V24.8H + add V15.8H, V15.8H, V23.8H + add V16.8H, V16.8H, V24.8H + mul V25.8H, V26.8H, V7.H[6] + mul V27.8H, V28.8H, V7.H[6] + sqrdmulh V23.8H, V26.8H, V5.H[6] + sqrdmulh V24.8H, V28.8H, V5.H[6] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V27.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + mul V25.8H, V9.8H, V7.H[7] + mul V26.8H, V10.8H, V7.H[7] + sqrdmulh V9.8H, V9.8H, V5.H[7] + sqrdmulh V10.8H, V10.8H, V5.H[7] + sqrdmlsh V9.8H, V25.8H, V8.H[0] + sqrdmlsh V10.8H, V26.8H, V8.H[0] + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V25.8H, V11.8H, V7.H[7] + mul V26.8H, V12.8H, V7.H[7] + sqrdmulh V11.8H, V11.8H, V5.H[7] + sqrdmulh V12.8H, V12.8H, V5.H[7] + sqrdmlsh V11.8H, V25.8H, V8.H[0] + sqrdmlsh V12.8H, V26.8H, V8.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V25.8H, V13.8H, V7.H[7] + mul V26.8H, V14.8H, V7.H[7] + sqrdmulh V13.8H, V13.8H, V5.H[7] + sqrdmulh V14.8H, V14.8H, V5.H[7] + sqrdmlsh V13.8H, V25.8H, V8.H[0] + sqrdmlsh V14.8H, V26.8H, V8.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V25.8H, V15.8H, V7.H[7] + mul V26.8H, V16.8H, V7.H[7] + sqrdmulh V15.8H, V15.8H, V5.H[7] + sqrdmulh V16.8H, V16.8H, V5.H[7] + sqrdmlsh V15.8H, V25.8H, V8.H[0] + sqrdmlsh V16.8H, V26.8H, V8.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + mul V25.8H, V17.8H, V7.H[7] + mul V26.8H, V18.8H, V7.H[7] + sqrdmulh V17.8H, V17.8H, V5.H[7] + sqrdmulh V18.8H, V18.8H, V5.H[7] + sqrdmlsh V17.8H, V25.8H, V8.H[0] + sqrdmlsh V18.8H, V26.8H, V8.H[0] + sshr V17.8H, V17.8H, #1 + sshr V18.8H, V18.8H, #1 + mul V25.8H, V19.8H, V7.H[7] + mul V26.8H, V20.8H, V7.H[7] + sqrdmulh V19.8H, V19.8H, V5.H[7] + sqrdmulh V20.8H, V20.8H, V5.H[7] + sqrdmlsh V19.8H, V25.8H, V8.H[0] + sqrdmlsh V20.8H, V26.8H, V8.H[0] + sshr V19.8H, V19.8H, #1 + sshr V20.8H, V20.8H, #1 + mul V25.8H, V21.8H, V7.H[7] + mul V26.8H, V22.8H, V7.H[7] + sqrdmulh V21.8H, V21.8H, V5.H[7] + sqrdmulh V22.8H, V22.8H, V5.H[7] + sqrdmlsh V21.8H, V25.8H, V8.H[0] + sqrdmlsh V22.8H, V26.8H, V8.H[0] + sshr V21.8H, V21.8H, #1 + sshr V22.8H, V22.8H, #1 + mul V25.8H, V23.8H, V7.H[7] + mul V26.8H, V24.8H, V7.H[7] + sqrdmulh V23.8H, V23.8H, V5.H[7] + sqrdmulh V24.8H, V24.8H, V5.H[7] + sqrdmlsh V23.8H, V25.8H, V8.H[0] + sqrdmlsh V24.8H, V26.8H, V8.H[0] + sshr V23.8H, V23.8H, #1 + sshr V24.8H, V24.8H, #1 + str Q9, [x0, #16] + str Q10, [x0, #48] + str Q11, [x0, #80] + str Q12, [x0, #112] + str Q13, [x0, #144] + str Q14, [x0, #176] + str Q15, [x0, #208] + str Q16, [x0, #240] + str Q17, [x1, #16] + str Q18, [x1, #48] + str Q19, [x1, #80] + str Q20, [x1, #112] + str Q21, [x1, #144] + str Q22, [x1, #176] + str Q23, [x1, #208] + str Q24, [x1, #240] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP ENDIF - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_zetas_mul DCW 0x08b2, 0xf74e, 0x01ae, 0xfe52, 0x022b, 0xfdd5, 0x034b, 0xfcb5 @@ -5344,675 +5344,675 @@ L_mlkem_aarch64_zetas_mul mlkem_basemul_mont PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x3, L_mlkem_aarch64_zetas_mul add x3, x3, L_mlkem_aarch64_zetas_mul adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts - ldr q1, [x4] - ldp q2, q3, [x1] - ldp q4, q5, [x1, #32] - ldp q6, q7, [x1, #64] - ldp q8, q9, [x1, #96] - ldp q10, q11, [x2] - ldp q12, q13, [x2, #32] - ldp q14, q15, [x2, #64] - ldp q16, q17, [x2, #96] - ldr q0, [x3] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0] - ldr q0, [x3, #16] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #32] - ldr q0, [x3, #32] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #64] - ldr q0, [x3, #48] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #96] - ldp q2, q3, [x1, #128] - ldp q4, q5, [x1, #160] - ldp q6, q7, [x1, #192] - ldp q8, q9, [x1, #224] - ldp q10, q11, [x2, #128] - ldp q12, q13, [x2, #160] - ldp q14, q15, [x2, #192] - ldp q16, q17, [x2, #224] - ldr q0, [x3, #64] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #128] - ldr q0, [x3, #80] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #160] - ldr q0, [x3, #96] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #192] - ldr q0, [x3, #112] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #224] - ldp q2, q3, [x1, #256] - ldp q4, q5, [x1, #288] - ldp q6, q7, [x1, #320] - ldp q8, q9, [x1, #352] - ldp q10, q11, [x2, #256] - ldp q12, q13, [x2, #288] - ldp q14, q15, [x2, #320] - ldp q16, q17, [x2, #352] - ldr q0, [x3, #128] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #256] - ldr q0, [x3, #144] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #288] - ldr q0, [x3, #160] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #320] - ldr q0, [x3, #176] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #352] - ldp q2, q3, [x1, #384] - ldp q4, q5, [x1, #416] - ldp q6, q7, [x1, #448] - ldp q8, q9, [x1, #480] - ldp q10, q11, [x2, #384] - ldp q12, q13, [x2, #416] - ldp q14, q15, [x2, #448] - ldp q16, q17, [x2, #480] - ldr q0, [x3, #192] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #384] - ldr q0, [x3, #208] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #416] - ldr q0, [x3, #224] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #448] - ldr q0, [x3, #240] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - stp q24, q25, [x0, #480] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q1, [x4] + ldp Q2, Q3, [x1] + ldp Q4, Q5, [x1, #32] + ldp Q6, Q7, [x1, #64] + ldp Q8, Q9, [x1, #96] + ldp Q10, Q11, [x2] + ldp Q12, Q13, [x2, #32] + ldp Q14, Q15, [x2, #64] + ldp Q16, Q17, [x2, #96] + ldr Q0, [x3] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0] + ldr Q0, [x3, #16] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #32] + ldr Q0, [x3, #32] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #64] + ldr Q0, [x3, #48] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #96] + ldp Q2, Q3, [x1, #128] + ldp Q4, Q5, [x1, #160] + ldp Q6, Q7, [x1, #192] + ldp Q8, Q9, [x1, #224] + ldp Q10, Q11, [x2, #128] + ldp Q12, Q13, [x2, #160] + ldp Q14, Q15, [x2, #192] + ldp Q16, Q17, [x2, #224] + ldr Q0, [x3, #64] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #128] + ldr Q0, [x3, #80] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #160] + ldr Q0, [x3, #96] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #192] + ldr Q0, [x3, #112] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #224] + ldp Q2, Q3, [x1, #256] + ldp Q4, Q5, [x1, #288] + ldp Q6, Q7, [x1, #320] + ldp Q8, Q9, [x1, #352] + ldp Q10, Q11, [x2, #256] + ldp Q12, Q13, [x2, #288] + ldp Q14, Q15, [x2, #320] + ldp Q16, Q17, [x2, #352] + ldr Q0, [x3, #128] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #256] + ldr Q0, [x3, #144] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #288] + ldr Q0, [x3, #160] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #320] + ldr Q0, [x3, #176] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #352] + ldp Q2, Q3, [x1, #384] + ldp Q4, Q5, [x1, #416] + ldp Q6, Q7, [x1, #448] + ldp Q8, Q9, [x1, #480] + ldp Q10, Q11, [x2, #384] + ldp Q12, Q13, [x2, #416] + ldp Q14, Q15, [x2, #448] + ldp Q16, Q17, [x2, #480] + ldr Q0, [x3, #192] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #384] + ldr Q0, [x3, #208] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #416] + ldr Q0, [x3, #224] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #448] + ldr Q0, [x3, #240] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + stp Q24, Q25, [x0, #480] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -6022,727 +6022,727 @@ mlkem_basemul_mont PROC mlkem_basemul_mont_add PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x3, L_mlkem_aarch64_zetas_mul add x3, x3, L_mlkem_aarch64_zetas_mul adrp x4, L_mlkem_aarch64_consts add x4, x4, L_mlkem_aarch64_consts - ldr q1, [x4] - ldp q2, q3, [x1] - ldp q4, q5, [x1, #32] - ldp q6, q7, [x1, #64] - ldp q8, q9, [x1, #96] - ldp q10, q11, [x2] - ldp q12, q13, [x2, #32] - ldp q14, q15, [x2, #64] - ldp q16, q17, [x2, #96] - ldp q28, q29, [x0] - ldr q0, [x3] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0] - ldp q28, q29, [x0, #32] - ldr q0, [x3, #16] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #32] - ldp q28, q29, [x0, #64] - ldr q0, [x3, #32] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #64] - ldp q28, q29, [x0, #96] - ldr q0, [x3, #48] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #96] - ldp q2, q3, [x1, #128] - ldp q4, q5, [x1, #160] - ldp q6, q7, [x1, #192] - ldp q8, q9, [x1, #224] - ldp q10, q11, [x2, #128] - ldp q12, q13, [x2, #160] - ldp q14, q15, [x2, #192] - ldp q16, q17, [x2, #224] - ldp q28, q29, [x0, #128] - ldr q0, [x3, #64] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #128] - ldp q28, q29, [x0, #160] - ldr q0, [x3, #80] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #160] - ldp q28, q29, [x0, #192] - ldr q0, [x3, #96] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #192] - ldp q28, q29, [x0, #224] - ldr q0, [x3, #112] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #224] - ldp q2, q3, [x1, #256] - ldp q4, q5, [x1, #288] - ldp q6, q7, [x1, #320] - ldp q8, q9, [x1, #352] - ldp q10, q11, [x2, #256] - ldp q12, q13, [x2, #288] - ldp q14, q15, [x2, #320] - ldp q16, q17, [x2, #352] - ldp q28, q29, [x0, #256] - ldr q0, [x3, #128] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #256] - ldp q28, q29, [x0, #288] - ldr q0, [x3, #144] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #288] - ldp q28, q29, [x0, #320] - ldr q0, [x3, #160] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #320] - ldp q28, q29, [x0, #352] - ldr q0, [x3, #176] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #352] - ldp q2, q3, [x1, #384] - ldp q4, q5, [x1, #416] - ldp q6, q7, [x1, #448] - ldp q8, q9, [x1, #480] - ldp q10, q11, [x2, #384] - ldp q12, q13, [x2, #416] - ldp q14, q15, [x2, #448] - ldp q16, q17, [x2, #480] - ldp q28, q29, [x0, #384] - ldr q0, [x3, #192] - uzp1 v18.8h, v2.8h, v3.8h - uzp2 v19.8h, v2.8h, v3.8h - uzp1 v20.8h, v10.8h, v11.8h - uzp2 v21.8h, v10.8h, v11.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #384] - ldp q28, q29, [x0, #416] - ldr q0, [x3, #208] - uzp1 v18.8h, v4.8h, v5.8h - uzp2 v19.8h, v4.8h, v5.8h - uzp1 v20.8h, v12.8h, v13.8h - uzp2 v21.8h, v12.8h, v13.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #416] - ldp q28, q29, [x0, #448] - ldr q0, [x3, #224] - uzp1 v18.8h, v6.8h, v7.8h - uzp2 v19.8h, v6.8h, v7.8h - uzp1 v20.8h, v14.8h, v15.8h - uzp2 v21.8h, v14.8h, v15.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #448] - ldp q28, q29, [x0, #480] - ldr q0, [x3, #240] - uzp1 v18.8h, v8.8h, v9.8h - uzp2 v19.8h, v8.8h, v9.8h - uzp1 v20.8h, v16.8h, v17.8h - uzp2 v21.8h, v16.8h, v17.8h - smull v26.4s, v18.4h, v20.4h - smull2 v27.4s, v18.8h, v20.8h - smull v23.4s, v19.4h, v21.4h - smull2 v24.4s, v19.8h, v21.8h - xtn v25.4h, v23.4s - xtn2 v25.8h, v24.4s - mul v25.8h, v25.8h, v1.h[1] - smlsl v23.4s, v25.4h, v1.h[0] - smlsl2 v24.4s, v25.8h, v1.h[0] - shrn v22.4h, v23.4s, #16 - shrn2 v22.8h, v24.4s, #16 - smlal v26.4s, v22.4h, v0.4h - smlal2 v27.4s, v22.8h, v0.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v22.4h, v26.4s, #16 - shrn2 v22.8h, v27.4s, #16 - smull v26.4s, v18.4h, v21.4h - smull2 v27.4s, v18.8h, v21.8h - smlal v26.4s, v19.4h, v20.4h - smlal2 v27.4s, v19.8h, v20.8h - xtn v24.4h, v26.4s - xtn2 v24.8h, v27.4s - mul v24.8h, v24.8h, v1.h[1] - smlsl v26.4s, v24.4h, v1.h[0] - smlsl2 v27.4s, v24.8h, v1.h[0] - shrn v23.4h, v26.4s, #16 - shrn2 v23.8h, v27.4s, #16 - zip1 v24.8h, v22.8h, v23.8h - zip2 v25.8h, v22.8h, v23.8h - add v28.8h, v28.8h, v24.8h - add v29.8h, v29.8h, v25.8h - stp q28, q29, [x0, #480] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q1, [x4] + ldp Q2, Q3, [x1] + ldp Q4, Q5, [x1, #32] + ldp Q6, Q7, [x1, #64] + ldp Q8, Q9, [x1, #96] + ldp Q10, Q11, [x2] + ldp Q12, Q13, [x2, #32] + ldp Q14, Q15, [x2, #64] + ldp Q16, Q17, [x2, #96] + ldp Q28, Q29, [x0] + ldr Q0, [x3] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0] + ldp Q28, Q29, [x0, #32] + ldr Q0, [x3, #16] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #32] + ldp Q28, Q29, [x0, #64] + ldr Q0, [x3, #32] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #64] + ldp Q28, Q29, [x0, #96] + ldr Q0, [x3, #48] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #96] + ldp Q2, Q3, [x1, #128] + ldp Q4, Q5, [x1, #160] + ldp Q6, Q7, [x1, #192] + ldp Q8, Q9, [x1, #224] + ldp Q10, Q11, [x2, #128] + ldp Q12, Q13, [x2, #160] + ldp Q14, Q15, [x2, #192] + ldp Q16, Q17, [x2, #224] + ldp Q28, Q29, [x0, #128] + ldr Q0, [x3, #64] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #128] + ldp Q28, Q29, [x0, #160] + ldr Q0, [x3, #80] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #160] + ldp Q28, Q29, [x0, #192] + ldr Q0, [x3, #96] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #192] + ldp Q28, Q29, [x0, #224] + ldr Q0, [x3, #112] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #224] + ldp Q2, Q3, [x1, #256] + ldp Q4, Q5, [x1, #288] + ldp Q6, Q7, [x1, #320] + ldp Q8, Q9, [x1, #352] + ldp Q10, Q11, [x2, #256] + ldp Q12, Q13, [x2, #288] + ldp Q14, Q15, [x2, #320] + ldp Q16, Q17, [x2, #352] + ldp Q28, Q29, [x0, #256] + ldr Q0, [x3, #128] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #256] + ldp Q28, Q29, [x0, #288] + ldr Q0, [x3, #144] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #288] + ldp Q28, Q29, [x0, #320] + ldr Q0, [x3, #160] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #320] + ldp Q28, Q29, [x0, #352] + ldr Q0, [x3, #176] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #352] + ldp Q2, Q3, [x1, #384] + ldp Q4, Q5, [x1, #416] + ldp Q6, Q7, [x1, #448] + ldp Q8, Q9, [x1, #480] + ldp Q10, Q11, [x2, #384] + ldp Q12, Q13, [x2, #416] + ldp Q14, Q15, [x2, #448] + ldp Q16, Q17, [x2, #480] + ldp Q28, Q29, [x0, #384] + ldr Q0, [x3, #192] + uzp1 V18.8H, V2.8H, V3.8H + uzp2 V19.8H, V2.8H, V3.8H + uzp1 V20.8H, V10.8H, V11.8H + uzp2 V21.8H, V10.8H, V11.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #384] + ldp Q28, Q29, [x0, #416] + ldr Q0, [x3, #208] + uzp1 V18.8H, V4.8H, V5.8H + uzp2 V19.8H, V4.8H, V5.8H + uzp1 V20.8H, V12.8H, V13.8H + uzp2 V21.8H, V12.8H, V13.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #416] + ldp Q28, Q29, [x0, #448] + ldr Q0, [x3, #224] + uzp1 V18.8H, V6.8H, V7.8H + uzp2 V19.8H, V6.8H, V7.8H + uzp1 V20.8H, V14.8H, V15.8H + uzp2 V21.8H, V14.8H, V15.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #448] + ldp Q28, Q29, [x0, #480] + ldr Q0, [x3, #240] + uzp1 V18.8H, V8.8H, V9.8H + uzp2 V19.8H, V8.8H, V9.8H + uzp1 V20.8H, V16.8H, V17.8H + uzp2 V21.8H, V16.8H, V17.8H + smull V26.4S, V18.4H, V20.4H + smull2 V27.4S, V18.8H, V20.8H + smull V23.4S, V19.4H, V21.4H + smull2 V24.4S, V19.8H, V21.8H + xtn V25.4H, V23.4S + xtn2 V25.8H, V24.4S + mul V25.8H, V25.8H, V1.H[1] + smlsl V23.4S, V25.4H, V1.H[0] + smlsl2 V24.4S, V25.8H, V1.H[0] + shrn V22.4H, V23.4S, #16 + shrn2 V22.8H, V24.4S, #16 + smlal V26.4S, V22.4H, V0.4H + smlal2 V27.4S, V22.8H, V0.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V22.4H, V26.4S, #16 + shrn2 V22.8H, V27.4S, #16 + smull V26.4S, V18.4H, V21.4H + smull2 V27.4S, V18.8H, V21.8H + smlal V26.4S, V19.4H, V20.4H + smlal2 V27.4S, V19.8H, V20.8H + xtn V24.4H, V26.4S + xtn2 V24.8H, V27.4S + mul V24.8H, V24.8H, V1.H[1] + smlsl V26.4S, V24.4H, V1.H[0] + smlsl2 V27.4S, V24.8H, V1.H[0] + shrn V23.4H, V26.4S, #16 + shrn2 V23.8H, V27.4S, #16 + zip1 V24.8H, V22.8H, V23.8H + zip2 V25.8H, V22.8H, V23.8H + add V28.8H, V28.8H, V24.8H + add V29.8H, V29.8H, V25.8H + stp Q28, Q29, [x0, #480] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_aarch64_q DCW 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01, 0x0d01 @@ -6752,163 +6752,163 @@ L_mlkem_aarch64_q mlkem_csubq_neon PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x1, L_mlkem_aarch64_q add x1, x1, L_mlkem_aarch64_q - ldr q20, [x1] - ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40 - ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - ld4 {v8.8h, v9.8h, v10.8h, v11.8h}, [x0], #0x40 - ld4 {v12.8h, v13.8h, v14.8h, v15.8h}, [x0], #0x40 + ldr Q20, [x1] + ld4 {V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40 + ld4 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + ld4 {V8.8H, V9.8H, V10.8H, V11.8H}, [x0], #0x40 + ld4 {V12.8H, V13.8H, V14.8H, V15.8H}, [x0], #0x40 sub x0, x0, #0x100 - sub v0.8h, v0.8h, v20.8h - sub v1.8h, v1.8h, v20.8h - sub v2.8h, v2.8h, v20.8h - sub v3.8h, v3.8h, v20.8h - sub v4.8h, v4.8h, v20.8h - sub v5.8h, v5.8h, v20.8h - sub v6.8h, v6.8h, v20.8h - sub v7.8h, v7.8h, v20.8h - sub v8.8h, v8.8h, v20.8h - sub v9.8h, v9.8h, v20.8h - sub v10.8h, v10.8h, v20.8h - sub v11.8h, v11.8h, v20.8h - sub v12.8h, v12.8h, v20.8h - sub v13.8h, v13.8h, v20.8h - sub v14.8h, v14.8h, v20.8h - sub v15.8h, v15.8h, v20.8h - sshr v16.8h, v0.8h, #15 - sshr v17.8h, v1.8h, #15 - sshr v18.8h, v2.8h, #15 - sshr v19.8h, v3.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v0.8h, v0.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - sshr v16.8h, v4.8h, #15 - sshr v17.8h, v5.8h, #15 - sshr v18.8h, v6.8h, #15 - sshr v19.8h, v7.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v4.8h, v4.8h, v16.8h - add v5.8h, v5.8h, v17.8h - add v6.8h, v6.8h, v18.8h - add v7.8h, v7.8h, v19.8h - sshr v16.8h, v8.8h, #15 - sshr v17.8h, v9.8h, #15 - sshr v18.8h, v10.8h, #15 - sshr v19.8h, v11.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v8.8h, v8.8h, v16.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - add v11.8h, v11.8h, v19.8h - sshr v16.8h, v12.8h, #15 - sshr v17.8h, v13.8h, #15 - sshr v18.8h, v14.8h, #15 - sshr v19.8h, v15.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v12.8h, v12.8h, v16.8h - add v13.8h, v13.8h, v17.8h - add v14.8h, v14.8h, v18.8h - add v15.8h, v15.8h, v19.8h - st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40 - st4 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - st4 {v8.8h, v9.8h, v10.8h, v11.8h}, [x0], #0x40 - st4 {v12.8h, v13.8h, v14.8h, v15.8h}, [x0], #0x40 - ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40 - ld4 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - ld4 {v8.8h, v9.8h, v10.8h, v11.8h}, [x0], #0x40 - ld4 {v12.8h, v13.8h, v14.8h, v15.8h}, [x0], #0x40 + sub V0.8H, V0.8H, V20.8H + sub V1.8H, V1.8H, V20.8H + sub V2.8H, V2.8H, V20.8H + sub V3.8H, V3.8H, V20.8H + sub V4.8H, V4.8H, V20.8H + sub V5.8H, V5.8H, V20.8H + sub V6.8H, V6.8H, V20.8H + sub V7.8H, V7.8H, V20.8H + sub V8.8H, V8.8H, V20.8H + sub V9.8H, V9.8H, V20.8H + sub V10.8H, V10.8H, V20.8H + sub V11.8H, V11.8H, V20.8H + sub V12.8H, V12.8H, V20.8H + sub V13.8H, V13.8H, V20.8H + sub V14.8H, V14.8H, V20.8H + sub V15.8H, V15.8H, V20.8H + sshr V16.8H, V0.8H, #15 + sshr V17.8H, V1.8H, #15 + sshr V18.8H, V2.8H, #15 + sshr V19.8H, V3.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V0.8H, V0.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + sshr V16.8H, V4.8H, #15 + sshr V17.8H, V5.8H, #15 + sshr V18.8H, V6.8H, #15 + sshr V19.8H, V7.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V4.8H, V4.8H, V16.8H + add V5.8H, V5.8H, V17.8H + add V6.8H, V6.8H, V18.8H + add V7.8H, V7.8H, V19.8H + sshr V16.8H, V8.8H, #15 + sshr V17.8H, V9.8H, #15 + sshr V18.8H, V10.8H, #15 + sshr V19.8H, V11.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V8.8H, V8.8H, V16.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + add V11.8H, V11.8H, V19.8H + sshr V16.8H, V12.8H, #15 + sshr V17.8H, V13.8H, #15 + sshr V18.8H, V14.8H, #15 + sshr V19.8H, V15.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V12.8H, V12.8H, V16.8H + add V13.8H, V13.8H, V17.8H + add V14.8H, V14.8H, V18.8H + add V15.8H, V15.8H, V19.8H + st4 {V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40 + st4 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + st4 {V8.8H, V9.8H, V10.8H, V11.8H}, [x0], #0x40 + st4 {V12.8H, V13.8H, V14.8H, V15.8H}, [x0], #0x40 + ld4 {V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40 + ld4 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + ld4 {V8.8H, V9.8H, V10.8H, V11.8H}, [x0], #0x40 + ld4 {V12.8H, V13.8H, V14.8H, V15.8H}, [x0], #0x40 sub x0, x0, #0x100 - sub v0.8h, v0.8h, v20.8h - sub v1.8h, v1.8h, v20.8h - sub v2.8h, v2.8h, v20.8h - sub v3.8h, v3.8h, v20.8h - sub v4.8h, v4.8h, v20.8h - sub v5.8h, v5.8h, v20.8h - sub v6.8h, v6.8h, v20.8h - sub v7.8h, v7.8h, v20.8h - sub v8.8h, v8.8h, v20.8h - sub v9.8h, v9.8h, v20.8h - sub v10.8h, v10.8h, v20.8h - sub v11.8h, v11.8h, v20.8h - sub v12.8h, v12.8h, v20.8h - sub v13.8h, v13.8h, v20.8h - sub v14.8h, v14.8h, v20.8h - sub v15.8h, v15.8h, v20.8h - sshr v16.8h, v0.8h, #15 - sshr v17.8h, v1.8h, #15 - sshr v18.8h, v2.8h, #15 - sshr v19.8h, v3.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v0.8h, v0.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - sshr v16.8h, v4.8h, #15 - sshr v17.8h, v5.8h, #15 - sshr v18.8h, v6.8h, #15 - sshr v19.8h, v7.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v4.8h, v4.8h, v16.8h - add v5.8h, v5.8h, v17.8h - add v6.8h, v6.8h, v18.8h - add v7.8h, v7.8h, v19.8h - sshr v16.8h, v8.8h, #15 - sshr v17.8h, v9.8h, #15 - sshr v18.8h, v10.8h, #15 - sshr v19.8h, v11.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v8.8h, v8.8h, v16.8h - add v9.8h, v9.8h, v17.8h - add v10.8h, v10.8h, v18.8h - add v11.8h, v11.8h, v19.8h - sshr v16.8h, v12.8h, #15 - sshr v17.8h, v13.8h, #15 - sshr v18.8h, v14.8h, #15 - sshr v19.8h, v15.8h, #15 - and v16.16b, v16.16b, v20.16b - and v17.16b, v17.16b, v20.16b - and v18.16b, v18.16b, v20.16b - and v19.16b, v19.16b, v20.16b - add v12.8h, v12.8h, v16.8h - add v13.8h, v13.8h, v17.8h - add v14.8h, v14.8h, v18.8h - add v15.8h, v15.8h, v19.8h - st4 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #0x40 - st4 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - st4 {v8.8h, v9.8h, v10.8h, v11.8h}, [x0], #0x40 - st4 {v12.8h, v13.8h, v14.8h, v15.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + sub V0.8H, V0.8H, V20.8H + sub V1.8H, V1.8H, V20.8H + sub V2.8H, V2.8H, V20.8H + sub V3.8H, V3.8H, V20.8H + sub V4.8H, V4.8H, V20.8H + sub V5.8H, V5.8H, V20.8H + sub V6.8H, V6.8H, V20.8H + sub V7.8H, V7.8H, V20.8H + sub V8.8H, V8.8H, V20.8H + sub V9.8H, V9.8H, V20.8H + sub V10.8H, V10.8H, V20.8H + sub V11.8H, V11.8H, V20.8H + sub V12.8H, V12.8H, V20.8H + sub V13.8H, V13.8H, V20.8H + sub V14.8H, V14.8H, V20.8H + sub V15.8H, V15.8H, V20.8H + sshr V16.8H, V0.8H, #15 + sshr V17.8H, V1.8H, #15 + sshr V18.8H, V2.8H, #15 + sshr V19.8H, V3.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V0.8H, V0.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + sshr V16.8H, V4.8H, #15 + sshr V17.8H, V5.8H, #15 + sshr V18.8H, V6.8H, #15 + sshr V19.8H, V7.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V4.8H, V4.8H, V16.8H + add V5.8H, V5.8H, V17.8H + add V6.8H, V6.8H, V18.8H + add V7.8H, V7.8H, V19.8H + sshr V16.8H, V8.8H, #15 + sshr V17.8H, V9.8H, #15 + sshr V18.8H, V10.8H, #15 + sshr V19.8H, V11.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V8.8H, V8.8H, V16.8H + add V9.8H, V9.8H, V17.8H + add V10.8H, V10.8H, V18.8H + add V11.8H, V11.8H, V19.8H + sshr V16.8H, V12.8H, #15 + sshr V17.8H, V13.8H, #15 + sshr V18.8H, V14.8H, #15 + sshr V19.8H, V15.8H, #15 + and V16.16B, V16.16B, V20.16B + and V17.16B, V17.16B, V20.16B + and V18.16B, V18.16B, V20.16B + and V19.16B, V19.16B, V20.16B + add V12.8H, V12.8H, V16.8H + add V13.8H, V13.8H, V17.8H + add V14.8H, V14.8H, V18.8H + add V15.8H, V15.8H, V19.8H + st4 {V0.8H, V1.8H, V2.8H, V3.8H}, [x0], #0x40 + st4 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + st4 {V8.8H, V9.8H, V10.8H, V11.8H}, [x0], #0x40 + st4 {V12.8H, V13.8H, V14.8H, V15.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -6918,173 +6918,173 @@ mlkem_csubq_neon PROC mlkem_add_reduce PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_consts add x2, x2, L_mlkem_aarch64_consts - ldr q0, [x2] - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + ldr Q0, [x2] + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -7094,213 +7094,213 @@ mlkem_add_reduce PROC mlkem_add3_reduce PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x3, L_mlkem_aarch64_consts add x3, x3, L_mlkem_aarch64_consts - ldr q0, [x3] - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 - ld4 {v17.8h, v18.8h, v19.8h, v20.8h}, [x2], #0x40 - ld4 {v21.8h, v22.8h, v23.8h, v24.8h}, [x2], #0x40 + ldr Q0, [x3] + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 + ld4 {V17.8H, V18.8H, V19.8H, V20.8H}, [x2], #0x40 + ld4 {V21.8H, V22.8H, V23.8H, V24.8H}, [x2], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - add v4.8h, v4.8h, v20.8h - add v5.8h, v5.8h, v21.8h - add v6.8h, v6.8h, v22.8h - add v7.8h, v7.8h, v23.8h - add v8.8h, v8.8h, v24.8h - sqdmulh v25.8h, v1.8h, v0.h[2] - sqdmulh v26.8h, v2.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v1.8h, v25.8h, v0.h[0] - mls v2.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v3.8h, v0.h[2] - sqdmulh v26.8h, v4.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v3.8h, v25.8h, v0.h[0] - mls v4.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v5.8h, v0.h[2] - sqdmulh v26.8h, v6.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v5.8h, v25.8h, v0.h[0] - mls v6.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v7.8h, v0.h[2] - sqdmulh v26.8h, v8.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v7.8h, v25.8h, v0.h[0] - mls v8.8h, v26.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 - ld4 {v17.8h, v18.8h, v19.8h, v20.8h}, [x2], #0x40 - ld4 {v21.8h, v22.8h, v23.8h, v24.8h}, [x2], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + add V4.8H, V4.8H, V20.8H + add V5.8H, V5.8H, V21.8H + add V6.8H, V6.8H, V22.8H + add V7.8H, V7.8H, V23.8H + add V8.8H, V8.8H, V24.8H + sqdmulh V25.8H, V1.8H, V0.H[2] + sqdmulh V26.8H, V2.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V1.8H, V25.8H, V0.H[0] + mls V2.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V3.8H, V0.H[2] + sqdmulh V26.8H, V4.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V3.8H, V25.8H, V0.H[0] + mls V4.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V5.8H, V0.H[2] + sqdmulh V26.8H, V6.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V5.8H, V25.8H, V0.H[0] + mls V6.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V7.8H, V0.H[2] + sqdmulh V26.8H, V8.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V7.8H, V25.8H, V0.H[0] + mls V8.8H, V26.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 + ld4 {V17.8H, V18.8H, V19.8H, V20.8H}, [x2], #0x40 + ld4 {V21.8H, V22.8H, V23.8H, V24.8H}, [x2], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - add v4.8h, v4.8h, v20.8h - add v5.8h, v5.8h, v21.8h - add v6.8h, v6.8h, v22.8h - add v7.8h, v7.8h, v23.8h - add v8.8h, v8.8h, v24.8h - sqdmulh v25.8h, v1.8h, v0.h[2] - sqdmulh v26.8h, v2.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v1.8h, v25.8h, v0.h[0] - mls v2.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v3.8h, v0.h[2] - sqdmulh v26.8h, v4.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v3.8h, v25.8h, v0.h[0] - mls v4.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v5.8h, v0.h[2] - sqdmulh v26.8h, v6.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v5.8h, v25.8h, v0.h[0] - mls v6.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v7.8h, v0.h[2] - sqdmulh v26.8h, v8.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v7.8h, v25.8h, v0.h[0] - mls v8.8h, v26.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 - ld4 {v17.8h, v18.8h, v19.8h, v20.8h}, [x2], #0x40 - ld4 {v21.8h, v22.8h, v23.8h, v24.8h}, [x2], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + add V4.8H, V4.8H, V20.8H + add V5.8H, V5.8H, V21.8H + add V6.8H, V6.8H, V22.8H + add V7.8H, V7.8H, V23.8H + add V8.8H, V8.8H, V24.8H + sqdmulh V25.8H, V1.8H, V0.H[2] + sqdmulh V26.8H, V2.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V1.8H, V25.8H, V0.H[0] + mls V2.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V3.8H, V0.H[2] + sqdmulh V26.8H, V4.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V3.8H, V25.8H, V0.H[0] + mls V4.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V5.8H, V0.H[2] + sqdmulh V26.8H, V6.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V5.8H, V25.8H, V0.H[0] + mls V6.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V7.8H, V0.H[2] + sqdmulh V26.8H, V8.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V7.8H, V25.8H, V0.H[0] + mls V8.8H, V26.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 + ld4 {V17.8H, V18.8H, V19.8H, V20.8H}, [x2], #0x40 + ld4 {V21.8H, V22.8H, V23.8H, V24.8H}, [x2], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - add v4.8h, v4.8h, v20.8h - add v5.8h, v5.8h, v21.8h - add v6.8h, v6.8h, v22.8h - add v7.8h, v7.8h, v23.8h - add v8.8h, v8.8h, v24.8h - sqdmulh v25.8h, v1.8h, v0.h[2] - sqdmulh v26.8h, v2.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v1.8h, v25.8h, v0.h[0] - mls v2.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v3.8h, v0.h[2] - sqdmulh v26.8h, v4.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v3.8h, v25.8h, v0.h[0] - mls v4.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v5.8h, v0.h[2] - sqdmulh v26.8h, v6.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v5.8h, v25.8h, v0.h[0] - mls v6.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v7.8h, v0.h[2] - sqdmulh v26.8h, v8.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v7.8h, v25.8h, v0.h[0] - mls v8.8h, v26.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 - ld4 {v17.8h, v18.8h, v19.8h, v20.8h}, [x2], #0x40 - ld4 {v21.8h, v22.8h, v23.8h, v24.8h}, [x2], #0x40 + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + add V4.8H, V4.8H, V20.8H + add V5.8H, V5.8H, V21.8H + add V6.8H, V6.8H, V22.8H + add V7.8H, V7.8H, V23.8H + add V8.8H, V8.8H, V24.8H + sqdmulh V25.8H, V1.8H, V0.H[2] + sqdmulh V26.8H, V2.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V1.8H, V25.8H, V0.H[0] + mls V2.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V3.8H, V0.H[2] + sqdmulh V26.8H, V4.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V3.8H, V25.8H, V0.H[0] + mls V4.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V5.8H, V0.H[2] + sqdmulh V26.8H, V6.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V5.8H, V25.8H, V0.H[0] + mls V6.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V7.8H, V0.H[2] + sqdmulh V26.8H, V8.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V7.8H, V25.8H, V0.H[0] + mls V8.8H, V26.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 + ld4 {V17.8H, V18.8H, V19.8H, V20.8H}, [x2], #0x40 + ld4 {V21.8H, V22.8H, V23.8H, V24.8H}, [x2], #0x40 sub x0, x0, #0x80 - add v1.8h, v1.8h, v9.8h - add v2.8h, v2.8h, v10.8h - add v3.8h, v3.8h, v11.8h - add v4.8h, v4.8h, v12.8h - add v5.8h, v5.8h, v13.8h - add v6.8h, v6.8h, v14.8h - add v7.8h, v7.8h, v15.8h - add v8.8h, v8.8h, v16.8h - add v1.8h, v1.8h, v17.8h - add v2.8h, v2.8h, v18.8h - add v3.8h, v3.8h, v19.8h - add v4.8h, v4.8h, v20.8h - add v5.8h, v5.8h, v21.8h - add v6.8h, v6.8h, v22.8h - add v7.8h, v7.8h, v23.8h - add v8.8h, v8.8h, v24.8h - sqdmulh v25.8h, v1.8h, v0.h[2] - sqdmulh v26.8h, v2.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v1.8h, v25.8h, v0.h[0] - mls v2.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v3.8h, v0.h[2] - sqdmulh v26.8h, v4.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v3.8h, v25.8h, v0.h[0] - mls v4.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v5.8h, v0.h[2] - sqdmulh v26.8h, v6.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v5.8h, v25.8h, v0.h[0] - mls v6.8h, v26.8h, v0.h[0] - sqdmulh v25.8h, v7.8h, v0.h[2] - sqdmulh v26.8h, v8.8h, v0.h[2] - sshr v25.8h, v25.8h, #11 - sshr v26.8h, v26.8h, #11 - mls v7.8h, v25.8h, v0.h[0] - mls v8.8h, v26.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + add V1.8H, V1.8H, V9.8H + add V2.8H, V2.8H, V10.8H + add V3.8H, V3.8H, V11.8H + add V4.8H, V4.8H, V12.8H + add V5.8H, V5.8H, V13.8H + add V6.8H, V6.8H, V14.8H + add V7.8H, V7.8H, V15.8H + add V8.8H, V8.8H, V16.8H + add V1.8H, V1.8H, V17.8H + add V2.8H, V2.8H, V18.8H + add V3.8H, V3.8H, V19.8H + add V4.8H, V4.8H, V20.8H + add V5.8H, V5.8H, V21.8H + add V6.8H, V6.8H, V22.8H + add V7.8H, V7.8H, V23.8H + add V8.8H, V8.8H, V24.8H + sqdmulh V25.8H, V1.8H, V0.H[2] + sqdmulh V26.8H, V2.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V1.8H, V25.8H, V0.H[0] + mls V2.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V3.8H, V0.H[2] + sqdmulh V26.8H, V4.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V3.8H, V25.8H, V0.H[0] + mls V4.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V5.8H, V0.H[2] + sqdmulh V26.8H, V6.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V5.8H, V25.8H, V0.H[0] + mls V6.8H, V26.8H, V0.H[0] + sqdmulh V25.8H, V7.8H, V0.H[2] + sqdmulh V26.8H, V8.8H, V0.H[2] + sshr V25.8H, V25.8H, #11 + sshr V26.8H, V26.8H, #11 + mls V7.8H, V25.8H, V0.H[0] + mls V8.8H, V26.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -7310,173 +7310,173 @@ mlkem_add3_reduce PROC mlkem_rsub_reduce PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_aarch64_consts add x2, x2, L_mlkem_aarch64_consts - ldr q0, [x2] - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + ldr Q0, [x2] + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - sub v1.8h, v9.8h, v1.8h - sub v2.8h, v10.8h, v2.8h - sub v3.8h, v11.8h, v3.8h - sub v4.8h, v12.8h, v4.8h - sub v5.8h, v13.8h, v5.8h - sub v6.8h, v14.8h, v6.8h - sub v7.8h, v15.8h, v7.8h - sub v8.8h, v16.8h, v8.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + sub V1.8H, V9.8H, V1.8H + sub V2.8H, V10.8H, V2.8H + sub V3.8H, V11.8H, V3.8H + sub V4.8H, V12.8H, V4.8H + sub V5.8H, V13.8H, V5.8H + sub V6.8H, V14.8H, V6.8H + sub V7.8H, V15.8H, V7.8H + sub V8.8H, V16.8H, V8.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - sub v1.8h, v9.8h, v1.8h - sub v2.8h, v10.8h, v2.8h - sub v3.8h, v11.8h, v3.8h - sub v4.8h, v12.8h, v4.8h - sub v5.8h, v13.8h, v5.8h - sub v6.8h, v14.8h, v6.8h - sub v7.8h, v15.8h, v7.8h - sub v8.8h, v16.8h, v8.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + sub V1.8H, V9.8H, V1.8H + sub V2.8H, V10.8H, V2.8H + sub V3.8H, V11.8H, V3.8H + sub V4.8H, V12.8H, V4.8H + sub V5.8H, V13.8H, V5.8H + sub V6.8H, V14.8H, V6.8H + sub V7.8H, V15.8H, V7.8H + sub V8.8H, V16.8H, V8.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - sub v1.8h, v9.8h, v1.8h - sub v2.8h, v10.8h, v2.8h - sub v3.8h, v11.8h, v3.8h - sub v4.8h, v12.8h, v4.8h - sub v5.8h, v13.8h, v5.8h - sub v6.8h, v14.8h, v6.8h - sub v7.8h, v15.8h, v7.8h - sub v8.8h, v16.8h, v8.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x1], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x1], #0x40 + sub V1.8H, V9.8H, V1.8H + sub V2.8H, V10.8H, V2.8H + sub V3.8H, V11.8H, V3.8H + sub V4.8H, V12.8H, V4.8H + sub V5.8H, V13.8H, V5.8H + sub V6.8H, V14.8H, V6.8H + sub V7.8H, V15.8H, V7.8H + sub V8.8H, V16.8H, V8.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x1], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x1], #0x40 sub x0, x0, #0x80 - sub v1.8h, v9.8h, v1.8h - sub v2.8h, v10.8h, v2.8h - sub v3.8h, v11.8h, v3.8h - sub v4.8h, v12.8h, v4.8h - sub v5.8h, v13.8h, v5.8h - sub v6.8h, v14.8h, v6.8h - sub v7.8h, v15.8h, v7.8h - sub v8.8h, v16.8h, v8.8h - sqdmulh v17.8h, v1.8h, v0.h[2] - sqdmulh v18.8h, v2.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v1.8h, v17.8h, v0.h[0] - mls v2.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v3.8h, v0.h[2] - sqdmulh v18.8h, v4.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v3.8h, v17.8h, v0.h[0] - mls v4.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v5.8h, v0.h[2] - sqdmulh v18.8h, v6.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v5.8h, v17.8h, v0.h[0] - mls v6.8h, v18.8h, v0.h[0] - sqdmulh v17.8h, v7.8h, v0.h[2] - sqdmulh v18.8h, v8.8h, v0.h[2] - sshr v17.8h, v17.8h, #11 - sshr v18.8h, v18.8h, #11 - mls v7.8h, v17.8h, v0.h[0] - mls v8.8h, v18.8h, v0.h[0] - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + sub V1.8H, V9.8H, V1.8H + sub V2.8H, V10.8H, V2.8H + sub V3.8H, V11.8H, V3.8H + sub V4.8H, V12.8H, V4.8H + sub V5.8H, V13.8H, V5.8H + sub V6.8H, V14.8H, V6.8H + sub V7.8H, V15.8H, V7.8H + sub V8.8H, V16.8H, V8.8H + sqdmulh V17.8H, V1.8H, V0.H[2] + sqdmulh V18.8H, V2.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V1.8H, V17.8H, V0.H[0] + mls V2.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V3.8H, V0.H[2] + sqdmulh V18.8H, V4.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V3.8H, V17.8H, V0.H[0] + mls V4.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V5.8H, V0.H[2] + sqdmulh V18.8H, V6.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V5.8H, V17.8H, V0.H[0] + mls V6.8H, V18.8H, V0.H[0] + sqdmulh V17.8H, V7.8H, V0.H[2] + sqdmulh V18.8H, V8.8H, V0.H[2] + sshr V17.8H, V17.8H, #11 + sshr V18.8H, V18.8H, #11 + mls V7.8H, V17.8H, V0.H[0] + mls V8.8H, V18.8H, V0.H[0] + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -7486,195 +7486,195 @@ mlkem_rsub_reduce PROC mlkem_to_mont PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x1, L_mlkem_aarch64_consts add x1, x1, L_mlkem_aarch64_consts - ldr q0, [x1] - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 + ldr Q0, [x1] + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 sub x0, x0, #0x100 - mul v17.8h, v1.8h, v0.h[4] - mul v18.8h, v2.8h, v0.h[4] - sqrdmulh v1.8h, v1.8h, v0.h[3] - sqrdmulh v2.8h, v2.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v1.8h, v1.8h, v17.8h - sub v2.8h, v2.8h, v18.8h - sshr v1.8h, v1.8h, #1 - sshr v2.8h, v2.8h, #1 - mul v17.8h, v3.8h, v0.h[4] - mul v18.8h, v4.8h, v0.h[4] - sqrdmulh v3.8h, v3.8h, v0.h[3] - sqrdmulh v4.8h, v4.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v3.8h, v3.8h, v17.8h - sub v4.8h, v4.8h, v18.8h - sshr v3.8h, v3.8h, #1 - sshr v4.8h, v4.8h, #1 - mul v17.8h, v5.8h, v0.h[4] - mul v18.8h, v6.8h, v0.h[4] - sqrdmulh v5.8h, v5.8h, v0.h[3] - sqrdmulh v6.8h, v6.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v5.8h, v5.8h, v17.8h - sub v6.8h, v6.8h, v18.8h - sshr v5.8h, v5.8h, #1 - sshr v6.8h, v6.8h, #1 - mul v17.8h, v7.8h, v0.h[4] - mul v18.8h, v8.8h, v0.h[4] - sqrdmulh v7.8h, v7.8h, v0.h[3] - sqrdmulh v8.8h, v8.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v7.8h, v7.8h, v17.8h - sub v8.8h, v8.8h, v18.8h - sshr v7.8h, v7.8h, #1 - sshr v8.8h, v8.8h, #1 - mul v17.8h, v9.8h, v0.h[4] - mul v18.8h, v10.8h, v0.h[4] - sqrdmulh v9.8h, v9.8h, v0.h[3] - sqrdmulh v10.8h, v10.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v9.8h, v9.8h, v17.8h - sub v10.8h, v10.8h, v18.8h - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v17.8h, v11.8h, v0.h[4] - mul v18.8h, v12.8h, v0.h[4] - sqrdmulh v11.8h, v11.8h, v0.h[3] - sqrdmulh v12.8h, v12.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v11.8h, v11.8h, v17.8h - sub v12.8h, v12.8h, v18.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v17.8h, v13.8h, v0.h[4] - mul v18.8h, v14.8h, v0.h[4] - sqrdmulh v13.8h, v13.8h, v0.h[3] - sqrdmulh v14.8h, v14.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v13.8h, v13.8h, v17.8h - sub v14.8h, v14.8h, v18.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v17.8h, v15.8h, v0.h[4] - mul v18.8h, v16.8h, v0.h[4] - sqrdmulh v15.8h, v15.8h, v0.h[3] - sqrdmulh v16.8h, v16.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v15.8h, v15.8h, v17.8h - sub v16.8h, v16.8h, v18.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - st4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - st4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 + mul V17.8H, V1.8H, V0.H[4] + mul V18.8H, V2.8H, V0.H[4] + sqrdmulh V1.8H, V1.8H, V0.H[3] + sqrdmulh V2.8H, V2.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V1.8H, V1.8H, V17.8H + sub V2.8H, V2.8H, V18.8H + sshr V1.8H, V1.8H, #1 + sshr V2.8H, V2.8H, #1 + mul V17.8H, V3.8H, V0.H[4] + mul V18.8H, V4.8H, V0.H[4] + sqrdmulh V3.8H, V3.8H, V0.H[3] + sqrdmulh V4.8H, V4.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V3.8H, V3.8H, V17.8H + sub V4.8H, V4.8H, V18.8H + sshr V3.8H, V3.8H, #1 + sshr V4.8H, V4.8H, #1 + mul V17.8H, V5.8H, V0.H[4] + mul V18.8H, V6.8H, V0.H[4] + sqrdmulh V5.8H, V5.8H, V0.H[3] + sqrdmulh V6.8H, V6.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V5.8H, V5.8H, V17.8H + sub V6.8H, V6.8H, V18.8H + sshr V5.8H, V5.8H, #1 + sshr V6.8H, V6.8H, #1 + mul V17.8H, V7.8H, V0.H[4] + mul V18.8H, V8.8H, V0.H[4] + sqrdmulh V7.8H, V7.8H, V0.H[3] + sqrdmulh V8.8H, V8.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V7.8H, V7.8H, V17.8H + sub V8.8H, V8.8H, V18.8H + sshr V7.8H, V7.8H, #1 + sshr V8.8H, V8.8H, #1 + mul V17.8H, V9.8H, V0.H[4] + mul V18.8H, V10.8H, V0.H[4] + sqrdmulh V9.8H, V9.8H, V0.H[3] + sqrdmulh V10.8H, V10.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V9.8H, V9.8H, V17.8H + sub V10.8H, V10.8H, V18.8H + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V17.8H, V11.8H, V0.H[4] + mul V18.8H, V12.8H, V0.H[4] + sqrdmulh V11.8H, V11.8H, V0.H[3] + sqrdmulh V12.8H, V12.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V11.8H, V11.8H, V17.8H + sub V12.8H, V12.8H, V18.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V17.8H, V13.8H, V0.H[4] + mul V18.8H, V14.8H, V0.H[4] + sqrdmulh V13.8H, V13.8H, V0.H[3] + sqrdmulh V14.8H, V14.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V13.8H, V13.8H, V17.8H + sub V14.8H, V14.8H, V18.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V17.8H, V15.8H, V0.H[4] + mul V18.8H, V16.8H, V0.H[4] + sqrdmulh V15.8H, V15.8H, V0.H[3] + sqrdmulh V16.8H, V16.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V15.8H, V15.8H, V17.8H + sub V16.8H, V16.8H, V18.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + st4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + st4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 sub x0, x0, #0x100 - mul v17.8h, v1.8h, v0.h[4] - mul v18.8h, v2.8h, v0.h[4] - sqrdmulh v1.8h, v1.8h, v0.h[3] - sqrdmulh v2.8h, v2.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v1.8h, v1.8h, v17.8h - sub v2.8h, v2.8h, v18.8h - sshr v1.8h, v1.8h, #1 - sshr v2.8h, v2.8h, #1 - mul v17.8h, v3.8h, v0.h[4] - mul v18.8h, v4.8h, v0.h[4] - sqrdmulh v3.8h, v3.8h, v0.h[3] - sqrdmulh v4.8h, v4.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v3.8h, v3.8h, v17.8h - sub v4.8h, v4.8h, v18.8h - sshr v3.8h, v3.8h, #1 - sshr v4.8h, v4.8h, #1 - mul v17.8h, v5.8h, v0.h[4] - mul v18.8h, v6.8h, v0.h[4] - sqrdmulh v5.8h, v5.8h, v0.h[3] - sqrdmulh v6.8h, v6.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v5.8h, v5.8h, v17.8h - sub v6.8h, v6.8h, v18.8h - sshr v5.8h, v5.8h, #1 - sshr v6.8h, v6.8h, #1 - mul v17.8h, v7.8h, v0.h[4] - mul v18.8h, v8.8h, v0.h[4] - sqrdmulh v7.8h, v7.8h, v0.h[3] - sqrdmulh v8.8h, v8.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v7.8h, v7.8h, v17.8h - sub v8.8h, v8.8h, v18.8h - sshr v7.8h, v7.8h, #1 - sshr v8.8h, v8.8h, #1 - mul v17.8h, v9.8h, v0.h[4] - mul v18.8h, v10.8h, v0.h[4] - sqrdmulh v9.8h, v9.8h, v0.h[3] - sqrdmulh v10.8h, v10.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v9.8h, v9.8h, v17.8h - sub v10.8h, v10.8h, v18.8h - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v17.8h, v11.8h, v0.h[4] - mul v18.8h, v12.8h, v0.h[4] - sqrdmulh v11.8h, v11.8h, v0.h[3] - sqrdmulh v12.8h, v12.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v11.8h, v11.8h, v17.8h - sub v12.8h, v12.8h, v18.8h - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v17.8h, v13.8h, v0.h[4] - mul v18.8h, v14.8h, v0.h[4] - sqrdmulh v13.8h, v13.8h, v0.h[3] - sqrdmulh v14.8h, v14.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v13.8h, v13.8h, v17.8h - sub v14.8h, v14.8h, v18.8h - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v17.8h, v15.8h, v0.h[4] - mul v18.8h, v16.8h, v0.h[4] - sqrdmulh v15.8h, v15.8h, v0.h[3] - sqrdmulh v16.8h, v16.8h, v0.h[3] - sqrdmulh v17.8h, v17.8h, v0.h[0] - sqrdmulh v18.8h, v18.8h, v0.h[0] - sub v15.8h, v15.8h, v17.8h - sub v16.8h, v16.8h, v18.8h - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - st4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - st4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + mul V17.8H, V1.8H, V0.H[4] + mul V18.8H, V2.8H, V0.H[4] + sqrdmulh V1.8H, V1.8H, V0.H[3] + sqrdmulh V2.8H, V2.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V1.8H, V1.8H, V17.8H + sub V2.8H, V2.8H, V18.8H + sshr V1.8H, V1.8H, #1 + sshr V2.8H, V2.8H, #1 + mul V17.8H, V3.8H, V0.H[4] + mul V18.8H, V4.8H, V0.H[4] + sqrdmulh V3.8H, V3.8H, V0.H[3] + sqrdmulh V4.8H, V4.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V3.8H, V3.8H, V17.8H + sub V4.8H, V4.8H, V18.8H + sshr V3.8H, V3.8H, #1 + sshr V4.8H, V4.8H, #1 + mul V17.8H, V5.8H, V0.H[4] + mul V18.8H, V6.8H, V0.H[4] + sqrdmulh V5.8H, V5.8H, V0.H[3] + sqrdmulh V6.8H, V6.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V5.8H, V5.8H, V17.8H + sub V6.8H, V6.8H, V18.8H + sshr V5.8H, V5.8H, #1 + sshr V6.8H, V6.8H, #1 + mul V17.8H, V7.8H, V0.H[4] + mul V18.8H, V8.8H, V0.H[4] + sqrdmulh V7.8H, V7.8H, V0.H[3] + sqrdmulh V8.8H, V8.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V7.8H, V7.8H, V17.8H + sub V8.8H, V8.8H, V18.8H + sshr V7.8H, V7.8H, #1 + sshr V8.8H, V8.8H, #1 + mul V17.8H, V9.8H, V0.H[4] + mul V18.8H, V10.8H, V0.H[4] + sqrdmulh V9.8H, V9.8H, V0.H[3] + sqrdmulh V10.8H, V10.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V9.8H, V9.8H, V17.8H + sub V10.8H, V10.8H, V18.8H + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V17.8H, V11.8H, V0.H[4] + mul V18.8H, V12.8H, V0.H[4] + sqrdmulh V11.8H, V11.8H, V0.H[3] + sqrdmulh V12.8H, V12.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V11.8H, V11.8H, V17.8H + sub V12.8H, V12.8H, V18.8H + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V17.8H, V13.8H, V0.H[4] + mul V18.8H, V14.8H, V0.H[4] + sqrdmulh V13.8H, V13.8H, V0.H[3] + sqrdmulh V14.8H, V14.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V13.8H, V13.8H, V17.8H + sub V14.8H, V14.8H, V18.8H + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V17.8H, V15.8H, V0.H[4] + mul V18.8H, V16.8H, V0.H[4] + sqrdmulh V15.8H, V15.8H, V0.H[3] + sqrdmulh V16.8H, V16.8H, V0.H[3] + sqrdmulh V17.8H, V17.8H, V0.H[0] + sqrdmulh V18.8H, V18.8H, V0.H[0] + sub V15.8H, V15.8H, V17.8H + sub V16.8H, V16.8H, V18.8H + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + st4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + st4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP @@ -7685,176 +7685,176 @@ mlkem_to_mont PROC mlkem_to_mont_sqrdmlsh PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x1, L_mlkem_aarch64_consts add x1, x1, L_mlkem_aarch64_consts - ldr q0, [x1] - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 + ldr Q0, [x1] + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 sub x0, x0, #0x100 - mul v17.8h, v1.8h, v0.h[4] - mul v18.8h, v2.8h, v0.h[4] - sqrdmulh v1.8h, v1.8h, v0.h[3] - sqrdmulh v2.8h, v2.8h, v0.h[3] - sqrdmlsh v1.8h, v17.8h, v0.h[0] - sqrdmlsh v2.8h, v18.8h, v0.h[0] - sshr v1.8h, v1.8h, #1 - sshr v2.8h, v2.8h, #1 - mul v17.8h, v3.8h, v0.h[4] - mul v18.8h, v4.8h, v0.h[4] - sqrdmulh v3.8h, v3.8h, v0.h[3] - sqrdmulh v4.8h, v4.8h, v0.h[3] - sqrdmlsh v3.8h, v17.8h, v0.h[0] - sqrdmlsh v4.8h, v18.8h, v0.h[0] - sshr v3.8h, v3.8h, #1 - sshr v4.8h, v4.8h, #1 - mul v17.8h, v5.8h, v0.h[4] - mul v18.8h, v6.8h, v0.h[4] - sqrdmulh v5.8h, v5.8h, v0.h[3] - sqrdmulh v6.8h, v6.8h, v0.h[3] - sqrdmlsh v5.8h, v17.8h, v0.h[0] - sqrdmlsh v6.8h, v18.8h, v0.h[0] - sshr v5.8h, v5.8h, #1 - sshr v6.8h, v6.8h, #1 - mul v17.8h, v7.8h, v0.h[4] - mul v18.8h, v8.8h, v0.h[4] - sqrdmulh v7.8h, v7.8h, v0.h[3] - sqrdmulh v8.8h, v8.8h, v0.h[3] - sqrdmlsh v7.8h, v17.8h, v0.h[0] - sqrdmlsh v8.8h, v18.8h, v0.h[0] - sshr v7.8h, v7.8h, #1 - sshr v8.8h, v8.8h, #1 - mul v17.8h, v9.8h, v0.h[4] - mul v18.8h, v10.8h, v0.h[4] - sqrdmulh v9.8h, v9.8h, v0.h[3] - sqrdmulh v10.8h, v10.8h, v0.h[3] - sqrdmlsh v9.8h, v17.8h, v0.h[0] - sqrdmlsh v10.8h, v18.8h, v0.h[0] - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v17.8h, v11.8h, v0.h[4] - mul v18.8h, v12.8h, v0.h[4] - sqrdmulh v11.8h, v11.8h, v0.h[3] - sqrdmulh v12.8h, v12.8h, v0.h[3] - sqrdmlsh v11.8h, v17.8h, v0.h[0] - sqrdmlsh v12.8h, v18.8h, v0.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v17.8h, v13.8h, v0.h[4] - mul v18.8h, v14.8h, v0.h[4] - sqrdmulh v13.8h, v13.8h, v0.h[3] - sqrdmulh v14.8h, v14.8h, v0.h[3] - sqrdmlsh v13.8h, v17.8h, v0.h[0] - sqrdmlsh v14.8h, v18.8h, v0.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v17.8h, v15.8h, v0.h[4] - mul v18.8h, v16.8h, v0.h[4] - sqrdmulh v15.8h, v15.8h, v0.h[3] - sqrdmulh v16.8h, v16.8h, v0.h[3] - sqrdmlsh v15.8h, v17.8h, v0.h[0] - sqrdmlsh v16.8h, v18.8h, v0.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - st4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - st4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 - ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - ld4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - ld4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - ld4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 + mul V17.8H, V1.8H, V0.H[4] + mul V18.8H, V2.8H, V0.H[4] + sqrdmulh V1.8H, V1.8H, V0.H[3] + sqrdmulh V2.8H, V2.8H, V0.H[3] + sqrdmlsh V1.8H, V17.8H, V0.H[0] + sqrdmlsh V2.8H, V18.8H, V0.H[0] + sshr V1.8H, V1.8H, #1 + sshr V2.8H, V2.8H, #1 + mul V17.8H, V3.8H, V0.H[4] + mul V18.8H, V4.8H, V0.H[4] + sqrdmulh V3.8H, V3.8H, V0.H[3] + sqrdmulh V4.8H, V4.8H, V0.H[3] + sqrdmlsh V3.8H, V17.8H, V0.H[0] + sqrdmlsh V4.8H, V18.8H, V0.H[0] + sshr V3.8H, V3.8H, #1 + sshr V4.8H, V4.8H, #1 + mul V17.8H, V5.8H, V0.H[4] + mul V18.8H, V6.8H, V0.H[4] + sqrdmulh V5.8H, V5.8H, V0.H[3] + sqrdmulh V6.8H, V6.8H, V0.H[3] + sqrdmlsh V5.8H, V17.8H, V0.H[0] + sqrdmlsh V6.8H, V18.8H, V0.H[0] + sshr V5.8H, V5.8H, #1 + sshr V6.8H, V6.8H, #1 + mul V17.8H, V7.8H, V0.H[4] + mul V18.8H, V8.8H, V0.H[4] + sqrdmulh V7.8H, V7.8H, V0.H[3] + sqrdmulh V8.8H, V8.8H, V0.H[3] + sqrdmlsh V7.8H, V17.8H, V0.H[0] + sqrdmlsh V8.8H, V18.8H, V0.H[0] + sshr V7.8H, V7.8H, #1 + sshr V8.8H, V8.8H, #1 + mul V17.8H, V9.8H, V0.H[4] + mul V18.8H, V10.8H, V0.H[4] + sqrdmulh V9.8H, V9.8H, V0.H[3] + sqrdmulh V10.8H, V10.8H, V0.H[3] + sqrdmlsh V9.8H, V17.8H, V0.H[0] + sqrdmlsh V10.8H, V18.8H, V0.H[0] + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V17.8H, V11.8H, V0.H[4] + mul V18.8H, V12.8H, V0.H[4] + sqrdmulh V11.8H, V11.8H, V0.H[3] + sqrdmulh V12.8H, V12.8H, V0.H[3] + sqrdmlsh V11.8H, V17.8H, V0.H[0] + sqrdmlsh V12.8H, V18.8H, V0.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V17.8H, V13.8H, V0.H[4] + mul V18.8H, V14.8H, V0.H[4] + sqrdmulh V13.8H, V13.8H, V0.H[3] + sqrdmulh V14.8H, V14.8H, V0.H[3] + sqrdmlsh V13.8H, V17.8H, V0.H[0] + sqrdmlsh V14.8H, V18.8H, V0.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V17.8H, V15.8H, V0.H[4] + mul V18.8H, V16.8H, V0.H[4] + sqrdmulh V15.8H, V15.8H, V0.H[3] + sqrdmulh V16.8H, V16.8H, V0.H[3] + sqrdmlsh V15.8H, V17.8H, V0.H[0] + sqrdmlsh V16.8H, V18.8H, V0.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + st4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + st4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 + ld4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + ld4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + ld4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + ld4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 sub x0, x0, #0x100 - mul v17.8h, v1.8h, v0.h[4] - mul v18.8h, v2.8h, v0.h[4] - sqrdmulh v1.8h, v1.8h, v0.h[3] - sqrdmulh v2.8h, v2.8h, v0.h[3] - sqrdmlsh v1.8h, v17.8h, v0.h[0] - sqrdmlsh v2.8h, v18.8h, v0.h[0] - sshr v1.8h, v1.8h, #1 - sshr v2.8h, v2.8h, #1 - mul v17.8h, v3.8h, v0.h[4] - mul v18.8h, v4.8h, v0.h[4] - sqrdmulh v3.8h, v3.8h, v0.h[3] - sqrdmulh v4.8h, v4.8h, v0.h[3] - sqrdmlsh v3.8h, v17.8h, v0.h[0] - sqrdmlsh v4.8h, v18.8h, v0.h[0] - sshr v3.8h, v3.8h, #1 - sshr v4.8h, v4.8h, #1 - mul v17.8h, v5.8h, v0.h[4] - mul v18.8h, v6.8h, v0.h[4] - sqrdmulh v5.8h, v5.8h, v0.h[3] - sqrdmulh v6.8h, v6.8h, v0.h[3] - sqrdmlsh v5.8h, v17.8h, v0.h[0] - sqrdmlsh v6.8h, v18.8h, v0.h[0] - sshr v5.8h, v5.8h, #1 - sshr v6.8h, v6.8h, #1 - mul v17.8h, v7.8h, v0.h[4] - mul v18.8h, v8.8h, v0.h[4] - sqrdmulh v7.8h, v7.8h, v0.h[3] - sqrdmulh v8.8h, v8.8h, v0.h[3] - sqrdmlsh v7.8h, v17.8h, v0.h[0] - sqrdmlsh v8.8h, v18.8h, v0.h[0] - sshr v7.8h, v7.8h, #1 - sshr v8.8h, v8.8h, #1 - mul v17.8h, v9.8h, v0.h[4] - mul v18.8h, v10.8h, v0.h[4] - sqrdmulh v9.8h, v9.8h, v0.h[3] - sqrdmulh v10.8h, v10.8h, v0.h[3] - sqrdmlsh v9.8h, v17.8h, v0.h[0] - sqrdmlsh v10.8h, v18.8h, v0.h[0] - sshr v9.8h, v9.8h, #1 - sshr v10.8h, v10.8h, #1 - mul v17.8h, v11.8h, v0.h[4] - mul v18.8h, v12.8h, v0.h[4] - sqrdmulh v11.8h, v11.8h, v0.h[3] - sqrdmulh v12.8h, v12.8h, v0.h[3] - sqrdmlsh v11.8h, v17.8h, v0.h[0] - sqrdmlsh v12.8h, v18.8h, v0.h[0] - sshr v11.8h, v11.8h, #1 - sshr v12.8h, v12.8h, #1 - mul v17.8h, v13.8h, v0.h[4] - mul v18.8h, v14.8h, v0.h[4] - sqrdmulh v13.8h, v13.8h, v0.h[3] - sqrdmulh v14.8h, v14.8h, v0.h[3] - sqrdmlsh v13.8h, v17.8h, v0.h[0] - sqrdmlsh v14.8h, v18.8h, v0.h[0] - sshr v13.8h, v13.8h, #1 - sshr v14.8h, v14.8h, #1 - mul v17.8h, v15.8h, v0.h[4] - mul v18.8h, v16.8h, v0.h[4] - sqrdmulh v15.8h, v15.8h, v0.h[3] - sqrdmulh v16.8h, v16.8h, v0.h[3] - sqrdmlsh v15.8h, v17.8h, v0.h[0] - sqrdmlsh v16.8h, v18.8h, v0.h[0] - sshr v15.8h, v15.8h, #1 - sshr v16.8h, v16.8h, #1 - st4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x0], #0x40 - st4 {v5.8h, v6.8h, v7.8h, v8.8h}, [x0], #0x40 - st4 {v9.8h, v10.8h, v11.8h, v12.8h}, [x0], #0x40 - st4 {v13.8h, v14.8h, v15.8h, v16.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + mul V17.8H, V1.8H, V0.H[4] + mul V18.8H, V2.8H, V0.H[4] + sqrdmulh V1.8H, V1.8H, V0.H[3] + sqrdmulh V2.8H, V2.8H, V0.H[3] + sqrdmlsh V1.8H, V17.8H, V0.H[0] + sqrdmlsh V2.8H, V18.8H, V0.H[0] + sshr V1.8H, V1.8H, #1 + sshr V2.8H, V2.8H, #1 + mul V17.8H, V3.8H, V0.H[4] + mul V18.8H, V4.8H, V0.H[4] + sqrdmulh V3.8H, V3.8H, V0.H[3] + sqrdmulh V4.8H, V4.8H, V0.H[3] + sqrdmlsh V3.8H, V17.8H, V0.H[0] + sqrdmlsh V4.8H, V18.8H, V0.H[0] + sshr V3.8H, V3.8H, #1 + sshr V4.8H, V4.8H, #1 + mul V17.8H, V5.8H, V0.H[4] + mul V18.8H, V6.8H, V0.H[4] + sqrdmulh V5.8H, V5.8H, V0.H[3] + sqrdmulh V6.8H, V6.8H, V0.H[3] + sqrdmlsh V5.8H, V17.8H, V0.H[0] + sqrdmlsh V6.8H, V18.8H, V0.H[0] + sshr V5.8H, V5.8H, #1 + sshr V6.8H, V6.8H, #1 + mul V17.8H, V7.8H, V0.H[4] + mul V18.8H, V8.8H, V0.H[4] + sqrdmulh V7.8H, V7.8H, V0.H[3] + sqrdmulh V8.8H, V8.8H, V0.H[3] + sqrdmlsh V7.8H, V17.8H, V0.H[0] + sqrdmlsh V8.8H, V18.8H, V0.H[0] + sshr V7.8H, V7.8H, #1 + sshr V8.8H, V8.8H, #1 + mul V17.8H, V9.8H, V0.H[4] + mul V18.8H, V10.8H, V0.H[4] + sqrdmulh V9.8H, V9.8H, V0.H[3] + sqrdmulh V10.8H, V10.8H, V0.H[3] + sqrdmlsh V9.8H, V17.8H, V0.H[0] + sqrdmlsh V10.8H, V18.8H, V0.H[0] + sshr V9.8H, V9.8H, #1 + sshr V10.8H, V10.8H, #1 + mul V17.8H, V11.8H, V0.H[4] + mul V18.8H, V12.8H, V0.H[4] + sqrdmulh V11.8H, V11.8H, V0.H[3] + sqrdmulh V12.8H, V12.8H, V0.H[3] + sqrdmlsh V11.8H, V17.8H, V0.H[0] + sqrdmlsh V12.8H, V18.8H, V0.H[0] + sshr V11.8H, V11.8H, #1 + sshr V12.8H, V12.8H, #1 + mul V17.8H, V13.8H, V0.H[4] + mul V18.8H, V14.8H, V0.H[4] + sqrdmulh V13.8H, V13.8H, V0.H[3] + sqrdmulh V14.8H, V14.8H, V0.H[3] + sqrdmlsh V13.8H, V17.8H, V0.H[0] + sqrdmlsh V14.8H, V18.8H, V0.H[0] + sshr V13.8H, V13.8H, #1 + sshr V14.8H, V14.8H, #1 + mul V17.8H, V15.8H, V0.H[4] + mul V18.8H, V16.8H, V0.H[4] + sqrdmulh V15.8H, V15.8H, V0.H[3] + sqrdmulh V16.8H, V16.8H, V0.H[3] + sqrdmlsh V15.8H, V17.8H, V0.H[0] + sqrdmlsh V16.8H, V18.8H, V0.H[0] + sshr V15.8H, V15.8H, #1 + sshr V16.8H, V16.8H, #1 + st4 {V1.8H, V2.8H, V3.8H, V4.8H}, [x0], #0x40 + st4 {V5.8H, V6.8H, V7.8H, V8.8H}, [x0], #0x40 + st4 {V9.8H, V10.8H, V11.8H, V12.8H}, [x0], #0x40 + st4 {V13.8H, V14.8H, V15.8H, V16.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP ENDIF - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_to_msg_low DCW 0x0373, 0x0373, 0x0373, 0x0373, 0x0373, 0x0373, 0x0373, 0x0373 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_to_msg_high DCW 0x09c0, 0x09c0, 0x09c0, 0x09c0, 0x09c0, 0x09c0, 0x09c0, 0x09c0 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_to_msg_bits DCW 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080 @@ -7864,231 +7864,231 @@ L_mlkem_to_msg_bits mlkem_to_msg_neon PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x2, L_mlkem_to_msg_low add x2, x2, L_mlkem_to_msg_low adrp x3, L_mlkem_to_msg_high add x3, x3, L_mlkem_to_msg_high adrp x4, L_mlkem_to_msg_bits add x4, x4, L_mlkem_to_msg_bits - ldr q0, [x2] - ldr q1, [x3] - ldr q26, [x4] - ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x1], #0x40 - ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [x1], #0x40 - cmge v10.8h, v2.8h, v0.8h - cmge v18.8h, v1.8h, v2.8h - cmge v11.8h, v3.8h, v0.8h - cmge v19.8h, v1.8h, v3.8h - cmge v12.8h, v4.8h, v0.8h - cmge v20.8h, v1.8h, v4.8h - cmge v13.8h, v5.8h, v0.8h - cmge v21.8h, v1.8h, v5.8h - cmge v14.8h, v6.8h, v0.8h - cmge v22.8h, v1.8h, v6.8h - cmge v15.8h, v7.8h, v0.8h - cmge v23.8h, v1.8h, v7.8h - cmge v16.8h, v8.8h, v0.8h - cmge v24.8h, v1.8h, v8.8h - cmge v17.8h, v9.8h, v0.8h - cmge v25.8h, v1.8h, v9.8h - and v18.16b, v18.16b, v10.16b - and v19.16b, v19.16b, v11.16b - and v20.16b, v20.16b, v12.16b - and v21.16b, v21.16b, v13.16b - and v22.16b, v22.16b, v14.16b - and v23.16b, v23.16b, v15.16b - and v24.16b, v24.16b, v16.16b - and v25.16b, v25.16b, v17.16b - and v18.16b, v18.16b, v26.16b - and v19.16b, v19.16b, v26.16b - and v20.16b, v20.16b, v26.16b - and v21.16b, v21.16b, v26.16b - and v22.16b, v22.16b, v26.16b - and v23.16b, v23.16b, v26.16b - and v24.16b, v24.16b, v26.16b - and v25.16b, v25.16b, v26.16b - addv h18, v18.8h - addv h19, v19.8h - addv h20, v20.8h - addv h21, v21.8h - addv h22, v22.8h - addv h23, v23.8h - addv h24, v24.8h - addv h25, v25.8h - ins v18.b[1], v19.b[0] - ins v18.b[2], v20.b[0] - ins v18.b[3], v21.b[0] - ins v18.b[4], v22.b[0] - ins v18.b[5], v23.b[0] - ins v18.b[6], v24.b[0] - ins v18.b[7], v25.b[0] - st1 {v18.8b}, [x0], #8 - ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x1], #0x40 - ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [x1], #0x40 - cmge v10.8h, v2.8h, v0.8h - cmge v18.8h, v1.8h, v2.8h - cmge v11.8h, v3.8h, v0.8h - cmge v19.8h, v1.8h, v3.8h - cmge v12.8h, v4.8h, v0.8h - cmge v20.8h, v1.8h, v4.8h - cmge v13.8h, v5.8h, v0.8h - cmge v21.8h, v1.8h, v5.8h - cmge v14.8h, v6.8h, v0.8h - cmge v22.8h, v1.8h, v6.8h - cmge v15.8h, v7.8h, v0.8h - cmge v23.8h, v1.8h, v7.8h - cmge v16.8h, v8.8h, v0.8h - cmge v24.8h, v1.8h, v8.8h - cmge v17.8h, v9.8h, v0.8h - cmge v25.8h, v1.8h, v9.8h - and v18.16b, v18.16b, v10.16b - and v19.16b, v19.16b, v11.16b - and v20.16b, v20.16b, v12.16b - and v21.16b, v21.16b, v13.16b - and v22.16b, v22.16b, v14.16b - and v23.16b, v23.16b, v15.16b - and v24.16b, v24.16b, v16.16b - and v25.16b, v25.16b, v17.16b - and v18.16b, v18.16b, v26.16b - and v19.16b, v19.16b, v26.16b - and v20.16b, v20.16b, v26.16b - and v21.16b, v21.16b, v26.16b - and v22.16b, v22.16b, v26.16b - and v23.16b, v23.16b, v26.16b - and v24.16b, v24.16b, v26.16b - and v25.16b, v25.16b, v26.16b - addv h18, v18.8h - addv h19, v19.8h - addv h20, v20.8h - addv h21, v21.8h - addv h22, v22.8h - addv h23, v23.8h - addv h24, v24.8h - addv h25, v25.8h - ins v18.b[1], v19.b[0] - ins v18.b[2], v20.b[0] - ins v18.b[3], v21.b[0] - ins v18.b[4], v22.b[0] - ins v18.b[5], v23.b[0] - ins v18.b[6], v24.b[0] - ins v18.b[7], v25.b[0] - st1 {v18.8b}, [x0], #8 - ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x1], #0x40 - ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [x1], #0x40 - cmge v10.8h, v2.8h, v0.8h - cmge v18.8h, v1.8h, v2.8h - cmge v11.8h, v3.8h, v0.8h - cmge v19.8h, v1.8h, v3.8h - cmge v12.8h, v4.8h, v0.8h - cmge v20.8h, v1.8h, v4.8h - cmge v13.8h, v5.8h, v0.8h - cmge v21.8h, v1.8h, v5.8h - cmge v14.8h, v6.8h, v0.8h - cmge v22.8h, v1.8h, v6.8h - cmge v15.8h, v7.8h, v0.8h - cmge v23.8h, v1.8h, v7.8h - cmge v16.8h, v8.8h, v0.8h - cmge v24.8h, v1.8h, v8.8h - cmge v17.8h, v9.8h, v0.8h - cmge v25.8h, v1.8h, v9.8h - and v18.16b, v18.16b, v10.16b - and v19.16b, v19.16b, v11.16b - and v20.16b, v20.16b, v12.16b - and v21.16b, v21.16b, v13.16b - and v22.16b, v22.16b, v14.16b - and v23.16b, v23.16b, v15.16b - and v24.16b, v24.16b, v16.16b - and v25.16b, v25.16b, v17.16b - and v18.16b, v18.16b, v26.16b - and v19.16b, v19.16b, v26.16b - and v20.16b, v20.16b, v26.16b - and v21.16b, v21.16b, v26.16b - and v22.16b, v22.16b, v26.16b - and v23.16b, v23.16b, v26.16b - and v24.16b, v24.16b, v26.16b - and v25.16b, v25.16b, v26.16b - addv h18, v18.8h - addv h19, v19.8h - addv h20, v20.8h - addv h21, v21.8h - addv h22, v22.8h - addv h23, v23.8h - addv h24, v24.8h - addv h25, v25.8h - ins v18.b[1], v19.b[0] - ins v18.b[2], v20.b[0] - ins v18.b[3], v21.b[0] - ins v18.b[4], v22.b[0] - ins v18.b[5], v23.b[0] - ins v18.b[6], v24.b[0] - ins v18.b[7], v25.b[0] - st1 {v18.8b}, [x0], #8 - ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x1], #0x40 - ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [x1], #0x40 - cmge v10.8h, v2.8h, v0.8h - cmge v18.8h, v1.8h, v2.8h - cmge v11.8h, v3.8h, v0.8h - cmge v19.8h, v1.8h, v3.8h - cmge v12.8h, v4.8h, v0.8h - cmge v20.8h, v1.8h, v4.8h - cmge v13.8h, v5.8h, v0.8h - cmge v21.8h, v1.8h, v5.8h - cmge v14.8h, v6.8h, v0.8h - cmge v22.8h, v1.8h, v6.8h - cmge v15.8h, v7.8h, v0.8h - cmge v23.8h, v1.8h, v7.8h - cmge v16.8h, v8.8h, v0.8h - cmge v24.8h, v1.8h, v8.8h - cmge v17.8h, v9.8h, v0.8h - cmge v25.8h, v1.8h, v9.8h - and v18.16b, v18.16b, v10.16b - and v19.16b, v19.16b, v11.16b - and v20.16b, v20.16b, v12.16b - and v21.16b, v21.16b, v13.16b - and v22.16b, v22.16b, v14.16b - and v23.16b, v23.16b, v15.16b - and v24.16b, v24.16b, v16.16b - and v25.16b, v25.16b, v17.16b - and v18.16b, v18.16b, v26.16b - and v19.16b, v19.16b, v26.16b - and v20.16b, v20.16b, v26.16b - and v21.16b, v21.16b, v26.16b - and v22.16b, v22.16b, v26.16b - and v23.16b, v23.16b, v26.16b - and v24.16b, v24.16b, v26.16b - and v25.16b, v25.16b, v26.16b - addv h18, v18.8h - addv h19, v19.8h - addv h20, v20.8h - addv h21, v21.8h - addv h22, v22.8h - addv h23, v23.8h - addv h24, v24.8h - addv h25, v25.8h - ins v18.b[1], v19.b[0] - ins v18.b[2], v20.b[0] - ins v18.b[3], v21.b[0] - ins v18.b[4], v22.b[0] - ins v18.b[5], v23.b[0] - ins v18.b[6], v24.b[0] - ins v18.b[7], v25.b[0] - st1 {v18.8b}, [x0], #8 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + ldr Q0, [x2] + ldr Q1, [x3] + ldr Q26, [x4] + ld1 {V2.8H, V3.8H, V4.8H, V5.8H}, [x1], #0x40 + ld1 {V6.8H, V7.8H, V8.8H, V9.8H}, [x1], #0x40 + cmge V10.8H, V2.8H, V0.8H + cmge V18.8H, V1.8H, V2.8H + cmge V11.8H, V3.8H, V0.8H + cmge V19.8H, V1.8H, V3.8H + cmge V12.8H, V4.8H, V0.8H + cmge V20.8H, V1.8H, V4.8H + cmge V13.8H, V5.8H, V0.8H + cmge V21.8H, V1.8H, V5.8H + cmge V14.8H, V6.8H, V0.8H + cmge V22.8H, V1.8H, V6.8H + cmge V15.8H, V7.8H, V0.8H + cmge V23.8H, V1.8H, V7.8H + cmge V16.8H, V8.8H, V0.8H + cmge V24.8H, V1.8H, V8.8H + cmge V17.8H, V9.8H, V0.8H + cmge V25.8H, V1.8H, V9.8H + and V18.16B, V18.16B, V10.16B + and V19.16B, V19.16B, V11.16B + and V20.16B, V20.16B, V12.16B + and V21.16B, V21.16B, V13.16B + and V22.16B, V22.16B, V14.16B + and V23.16B, V23.16B, V15.16B + and V24.16B, V24.16B, V16.16B + and V25.16B, V25.16B, V17.16B + and V18.16B, V18.16B, V26.16B + and V19.16B, V19.16B, V26.16B + and V20.16B, V20.16B, V26.16B + and V21.16B, V21.16B, V26.16B + and V22.16B, V22.16B, V26.16B + and V23.16B, V23.16B, V26.16B + and V24.16B, V24.16B, V26.16B + and V25.16B, V25.16B, V26.16B + addv H18, V18.8H + addv H19, V19.8H + addv H20, V20.8H + addv H21, V21.8H + addv H22, V22.8H + addv H23, V23.8H + addv H24, V24.8H + addv H25, V25.8H + ins V18.B[1], V19.B[0] + ins V18.B[2], V20.B[0] + ins V18.B[3], V21.B[0] + ins V18.B[4], V22.B[0] + ins V18.B[5], V23.B[0] + ins V18.B[6], V24.B[0] + ins V18.B[7], V25.B[0] + st1 {V18.8B}, [x0], #8 + ld1 {V2.8H, V3.8H, V4.8H, V5.8H}, [x1], #0x40 + ld1 {V6.8H, V7.8H, V8.8H, V9.8H}, [x1], #0x40 + cmge V10.8H, V2.8H, V0.8H + cmge V18.8H, V1.8H, V2.8H + cmge V11.8H, V3.8H, V0.8H + cmge V19.8H, V1.8H, V3.8H + cmge V12.8H, V4.8H, V0.8H + cmge V20.8H, V1.8H, V4.8H + cmge V13.8H, V5.8H, V0.8H + cmge V21.8H, V1.8H, V5.8H + cmge V14.8H, V6.8H, V0.8H + cmge V22.8H, V1.8H, V6.8H + cmge V15.8H, V7.8H, V0.8H + cmge V23.8H, V1.8H, V7.8H + cmge V16.8H, V8.8H, V0.8H + cmge V24.8H, V1.8H, V8.8H + cmge V17.8H, V9.8H, V0.8H + cmge V25.8H, V1.8H, V9.8H + and V18.16B, V18.16B, V10.16B + and V19.16B, V19.16B, V11.16B + and V20.16B, V20.16B, V12.16B + and V21.16B, V21.16B, V13.16B + and V22.16B, V22.16B, V14.16B + and V23.16B, V23.16B, V15.16B + and V24.16B, V24.16B, V16.16B + and V25.16B, V25.16B, V17.16B + and V18.16B, V18.16B, V26.16B + and V19.16B, V19.16B, V26.16B + and V20.16B, V20.16B, V26.16B + and V21.16B, V21.16B, V26.16B + and V22.16B, V22.16B, V26.16B + and V23.16B, V23.16B, V26.16B + and V24.16B, V24.16B, V26.16B + and V25.16B, V25.16B, V26.16B + addv H18, V18.8H + addv H19, V19.8H + addv H20, V20.8H + addv H21, V21.8H + addv H22, V22.8H + addv H23, V23.8H + addv H24, V24.8H + addv H25, V25.8H + ins V18.B[1], V19.B[0] + ins V18.B[2], V20.B[0] + ins V18.B[3], V21.B[0] + ins V18.B[4], V22.B[0] + ins V18.B[5], V23.B[0] + ins V18.B[6], V24.B[0] + ins V18.B[7], V25.B[0] + st1 {V18.8B}, [x0], #8 + ld1 {V2.8H, V3.8H, V4.8H, V5.8H}, [x1], #0x40 + ld1 {V6.8H, V7.8H, V8.8H, V9.8H}, [x1], #0x40 + cmge V10.8H, V2.8H, V0.8H + cmge V18.8H, V1.8H, V2.8H + cmge V11.8H, V3.8H, V0.8H + cmge V19.8H, V1.8H, V3.8H + cmge V12.8H, V4.8H, V0.8H + cmge V20.8H, V1.8H, V4.8H + cmge V13.8H, V5.8H, V0.8H + cmge V21.8H, V1.8H, V5.8H + cmge V14.8H, V6.8H, V0.8H + cmge V22.8H, V1.8H, V6.8H + cmge V15.8H, V7.8H, V0.8H + cmge V23.8H, V1.8H, V7.8H + cmge V16.8H, V8.8H, V0.8H + cmge V24.8H, V1.8H, V8.8H + cmge V17.8H, V9.8H, V0.8H + cmge V25.8H, V1.8H, V9.8H + and V18.16B, V18.16B, V10.16B + and V19.16B, V19.16B, V11.16B + and V20.16B, V20.16B, V12.16B + and V21.16B, V21.16B, V13.16B + and V22.16B, V22.16B, V14.16B + and V23.16B, V23.16B, V15.16B + and V24.16B, V24.16B, V16.16B + and V25.16B, V25.16B, V17.16B + and V18.16B, V18.16B, V26.16B + and V19.16B, V19.16B, V26.16B + and V20.16B, V20.16B, V26.16B + and V21.16B, V21.16B, V26.16B + and V22.16B, V22.16B, V26.16B + and V23.16B, V23.16B, V26.16B + and V24.16B, V24.16B, V26.16B + and V25.16B, V25.16B, V26.16B + addv H18, V18.8H + addv H19, V19.8H + addv H20, V20.8H + addv H21, V21.8H + addv H22, V22.8H + addv H23, V23.8H + addv H24, V24.8H + addv H25, V25.8H + ins V18.B[1], V19.B[0] + ins V18.B[2], V20.B[0] + ins V18.B[3], V21.B[0] + ins V18.B[4], V22.B[0] + ins V18.B[5], V23.B[0] + ins V18.B[6], V24.B[0] + ins V18.B[7], V25.B[0] + st1 {V18.8B}, [x0], #8 + ld1 {V2.8H, V3.8H, V4.8H, V5.8H}, [x1], #0x40 + ld1 {V6.8H, V7.8H, V8.8H, V9.8H}, [x1], #0x40 + cmge V10.8H, V2.8H, V0.8H + cmge V18.8H, V1.8H, V2.8H + cmge V11.8H, V3.8H, V0.8H + cmge V19.8H, V1.8H, V3.8H + cmge V12.8H, V4.8H, V0.8H + cmge V20.8H, V1.8H, V4.8H + cmge V13.8H, V5.8H, V0.8H + cmge V21.8H, V1.8H, V5.8H + cmge V14.8H, V6.8H, V0.8H + cmge V22.8H, V1.8H, V6.8H + cmge V15.8H, V7.8H, V0.8H + cmge V23.8H, V1.8H, V7.8H + cmge V16.8H, V8.8H, V0.8H + cmge V24.8H, V1.8H, V8.8H + cmge V17.8H, V9.8H, V0.8H + cmge V25.8H, V1.8H, V9.8H + and V18.16B, V18.16B, V10.16B + and V19.16B, V19.16B, V11.16B + and V20.16B, V20.16B, V12.16B + and V21.16B, V21.16B, V13.16B + and V22.16B, V22.16B, V14.16B + and V23.16B, V23.16B, V15.16B + and V24.16B, V24.16B, V16.16B + and V25.16B, V25.16B, V17.16B + and V18.16B, V18.16B, V26.16B + and V19.16B, V19.16B, V26.16B + and V20.16B, V20.16B, V26.16B + and V21.16B, V21.16B, V26.16B + and V22.16B, V22.16B, V26.16B + and V23.16B, V23.16B, V26.16B + and V24.16B, V24.16B, V26.16B + and V25.16B, V25.16B, V26.16B + addv H18, V18.8H + addv H19, V19.8H + addv H20, V20.8H + addv H21, V21.8H + addv H22, V22.8H + addv H23, V23.8H + addv H24, V24.8H + addv H25, V25.8H + ins V18.B[1], V19.B[0] + ins V18.B[2], V20.B[0] + ins V18.B[3], V21.B[0] + ins V18.B[4], V22.B[0] + ins V18.B[5], V23.B[0] + ins V18.B[6], V24.B[0] + ins V18.B[7], V25.B[0] + st1 {V18.8B}, [x0], #8 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_from_msg_q1half DCW 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681, 0x0681 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_from_msg_bits DCB 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 @@ -8099,153 +8099,153 @@ L_mlkem_from_msg_bits mlkem_from_msg_neon PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] adrp x2, L_mlkem_from_msg_q1half add x2, x2, L_mlkem_from_msg_q1half adrp x3, L_mlkem_from_msg_bits add x3, x3, L_mlkem_from_msg_bits - ld1 {v2.16b, v3.16b}, [x1] - ldr q1, [x2] - ldr q0, [x3] - dup v4.8b, v2.b[0] - dup v5.8b, v2.b[1] - dup v6.8b, v2.b[2] - dup v7.8b, v2.b[3] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v2.b[4] - dup v5.8b, v2.b[5] - dup v6.8b, v2.b[6] - dup v7.8b, v2.b[7] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v2.b[8] - dup v5.8b, v2.b[9] - dup v6.8b, v2.b[10] - dup v7.8b, v2.b[11] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v2.b[12] - dup v5.8b, v2.b[13] - dup v6.8b, v2.b[14] - dup v7.8b, v2.b[15] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v3.b[0] - dup v5.8b, v3.b[1] - dup v6.8b, v3.b[2] - dup v7.8b, v3.b[3] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v3.b[4] - dup v5.8b, v3.b[5] - dup v6.8b, v3.b[6] - dup v7.8b, v3.b[7] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v3.b[8] - dup v5.8b, v3.b[9] - dup v6.8b, v3.b[10] - dup v7.8b, v3.b[11] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - dup v4.8b, v3.b[12] - dup v5.8b, v3.b[13] - dup v6.8b, v3.b[14] - dup v7.8b, v3.b[15] - cmtst v4.8b, v4.8b, v0.8b - cmtst v5.8b, v5.8b, v0.8b - cmtst v6.8b, v6.8b, v0.8b - cmtst v7.8b, v7.8b, v0.8b - zip1 v4.16b, v4.16b, v4.16b - zip1 v5.16b, v5.16b, v5.16b - zip1 v6.16b, v6.16b, v6.16b - zip1 v7.16b, v7.16b, v7.16b - and v4.16b, v4.16b, v1.16b - and v5.16b, v5.16b, v1.16b - and v6.16b, v6.16b, v1.16b - and v7.16b, v7.16b, v1.16b - st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #0x40 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] + ld1 {V2.16B, V3.16B}, [x1] + ldr Q1, [x2] + ldr Q0, [x3] + dup V4.8B, V2.B[0] + dup V5.8B, V2.B[1] + dup V6.8B, V2.B[2] + dup V7.8B, V2.B[3] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V2.B[4] + dup V5.8B, V2.B[5] + dup V6.8B, V2.B[6] + dup V7.8B, V2.B[7] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V2.B[8] + dup V5.8B, V2.B[9] + dup V6.8B, V2.B[10] + dup V7.8B, V2.B[11] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V2.B[12] + dup V5.8B, V2.B[13] + dup V6.8B, V2.B[14] + dup V7.8B, V2.B[15] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V3.B[0] + dup V5.8B, V3.B[1] + dup V6.8B, V3.B[2] + dup V7.8B, V3.B[3] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V3.B[4] + dup V5.8B, V3.B[5] + dup V6.8B, V3.B[6] + dup V7.8B, V3.B[7] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V3.B[8] + dup V5.8B, V3.B[9] + dup V6.8B, V3.B[10] + dup V7.8B, V3.B[11] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + dup V4.8B, V3.B[12] + dup V5.8B, V3.B[13] + dup V6.8B, V3.B[14] + dup V7.8B, V3.B[15] + cmtst V4.8B, V4.8B, V0.8B + cmtst V5.8B, V5.8B, V0.8B + cmtst V6.8B, V6.8B, V0.8B + cmtst V7.8B, V7.8B, V0.8B + zip1 V4.16B, V4.16B, V4.16B + zip1 V5.16B, V5.16B, V5.16B + zip1 V6.16B, V6.16B, V6.16B + zip1 V7.16B, V7.16B, V7.16B + and V4.16B, V4.16B, V1.16B + and V5.16B, V5.16B, V1.16B + and V6.16B, V6.16B, V1.16B + and V7.16B, V7.16B, V1.16B + st1 {V4.8H, V5.8H, V6.8H, V7.8H}, [x0], #0x40 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] ldp x29, x30, [sp], #48 ret ENDP @@ -8255,277 +8255,277 @@ mlkem_from_msg_neon PROC mlkem_cmp_neon PROC stp x29, x30, [sp, #-48]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v8.16b, v0.16b, v4.16b - eor v9.16b, v1.16b, v5.16b - eor v10.16b, v2.16b, v6.16b - eor v11.16b, v3.16b, v7.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V8.16B, V0.16B, V4.16B + eor V9.16B, V1.16B, V5.16B + eor V10.16B, V2.16B, V6.16B + eor V11.16B, V3.16B, V7.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B subs w2, w2, #0x300 beq L_mlkem_aarch64_cmp_neon_done - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B subs w2, w2, #0x140 beq L_mlkem_aarch64_cmp_neon_done - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld4 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #0x40 - ld4 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b - orr v10.16b, v10.16b, v2.16b - orr v11.16b, v11.16b, v3.16b - ld2 {v0.16b, v1.16b}, [x0] - ld2 {v4.16b, v5.16b}, [x1] - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - orr v8.16b, v8.16b, v0.16b - orr v9.16b, v9.16b, v1.16b + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld4 {V0.16B, V1.16B, V2.16B, V3.16B}, [x0], #0x40 + ld4 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + eor V2.16B, V2.16B, V6.16B + eor V3.16B, V3.16B, V7.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B + orr V10.16B, V10.16B, V2.16B + orr V11.16B, V11.16B, V3.16B + ld2 {V0.16B, V1.16B}, [x0] + ld2 {V4.16B, V5.16B}, [x1] + eor V0.16B, V0.16B, V4.16B + eor V1.16B, V1.16B, V5.16B + orr V8.16B, V8.16B, V0.16B + orr V9.16B, V9.16B, V1.16B L_mlkem_aarch64_cmp_neon_done - orr v8.16b, v8.16b, v9.16b - orr v10.16b, v10.16b, v11.16b - orr v8.16b, v8.16b, v10.16b - ext v9.16b, v8.16b, v8.16b, #8 - orr v8.16b, v8.16b, v9.16b - mov x0, v8.d[0] + orr V8.16B, V8.16B, V9.16B + orr V10.16B, V10.16B, V11.16B + orr V8.16B, V8.16B, V10.16B + ext8 V9.16B, V8.16B, V8.16B, #8 + orr V8.16B, V8.16B, V9.16B + mov x0, V8.D[0] subs x0, x0, xzr csetm w0, ne - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] ldp x29, x30, [sp], #48 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_rej_uniform_mask DCW 0x0fff, 0x0fff, 0x0fff, 0x0fff, 0x0fff, 0x0fff, 0x0fff, 0x0fff - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_rej_uniform_bits DCW 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_mlkem_rej_uniform_indices DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff @@ -9046,9 +9046,9 @@ L_mlkem_rej_uniform_indices mlkem_rej_uniform_neon PROC stp x29, x30, [sp, #-64]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] adrp x4, L_mlkem_rej_uniform_mask add x4, x4, L_mlkem_rej_uniform_mask adrp x5, L_mlkem_aarch64_q @@ -9057,58 +9057,58 @@ mlkem_rej_uniform_neon PROC add x6, x6, L_mlkem_rej_uniform_bits adrp x7, L_mlkem_rej_uniform_indices add x7, x7, L_mlkem_rej_uniform_indices - eor v1.16b, v1.16b, v1.16b - eor v12.16b, v12.16b, v12.16b - eor v13.16b, v13.16b, v13.16b + eor V1.16B, V1.16B, V1.16B + eor V12.16B, V12.16B, V12.16B + eor V13.16B, V13.16B, V13.16B eor x12, x12, x12 - eor v10.16b, v10.16b, v10.16b - eor v11.16b, v11.16b, v11.16b + eor V10.16B, V10.16B, V10.16B + eor V11.16B, V11.16B, V11.16B mov x13, #0xd01 - ldr q0, [x4] - ldr q3, [x5] - ldr q2, [x6] + ldr Q0, [x4] + ldr Q3, [x5] + ldr Q2, [x6] subs wzr, w1, #0 beq L_mlkem_rej_uniform_done subs wzr, w1, #16 blt L_mlkem_rej_uniform_loop_4 L_mlkem_rej_uniform_loop_16 - ld3 {v4.8b, v5.8b, v6.8b}, [x2], #24 - zip1 v4.16b, v4.16b, v1.16b - zip1 v5.16b, v5.16b, v1.16b - zip1 v6.16b, v6.16b, v1.16b - shl v7.8h, v5.8h, #8 - ushr v8.8h, v5.8h, #4 - shl v6.8h, v6.8h, #4 - orr v4.16b, v4.16b, v7.16b - orr v5.16b, v8.16b, v6.16b - and v7.16b, v4.16b, v0.16b - and v8.16b, v5.16b, v0.16b - zip1 v4.8h, v7.8h, v8.8h - zip2 v5.8h, v7.8h, v8.8h - cmgt v7.8h, v3.8h, v4.8h - cmgt v8.8h, v3.8h, v5.8h - ushr v12.8h, v7.8h, #15 - ushr v13.8h, v8.8h, #15 - addv h12, v12.8h - addv h13, v13.8h - mov x10, v12.d[0] - mov x11, v13.d[0] - and v10.16b, v7.16b, v2.16b - and v11.16b, v8.16b, v2.16b - addv h10, v10.8h - addv h11, v11.8h - mov w8, v10.s[0] - mov w9, v11.s[0] + ld3 {V4.8B, V5.8B, V6.8B}, [x2], #24 + zip1 V4.16B, V4.16B, V1.16B + zip1 V5.16B, V5.16B, V1.16B + zip1 V6.16B, V6.16B, V1.16B + shl V7.8H, V5.8H, #8 + ushr V8.8H, V5.8H, #4 + shl V6.8H, V6.8H, #4 + orr V4.16B, V4.16B, V7.16B + orr V5.16B, V8.16B, V6.16B + and V7.16B, V4.16B, V0.16B + and V8.16B, V5.16B, V0.16B + zip1 V4.8H, V7.8H, V8.8H + zip2 V5.8H, V7.8H, V8.8H + cmgt V7.8H, V3.8H, V4.8H + cmgt V8.8H, V3.8H, V5.8H + ushr V12.8H, V7.8H, #15 + ushr V13.8H, V8.8H, #15 + addv H12, V12.8H + addv H13, V13.8H + mov x10, V12.D[0] + mov x11, V13.D[0] + and V10.16B, V7.16B, V2.16B + and V11.16B, V8.16B, V2.16B + addv H10, V10.8H + addv H11, V11.8H + mov w8, V10.S[0] + mov w9, V11.S[0] lsl w8, w8, #4 lsl w9, w9, #4 - ldr q10, [x7, x8] - ldr q11, [x7, x9] - tbl v7.16b, {v4.16b}, v10.16b - tbl v8.16b, {v5.16b}, v11.16b - str q7, [x0] + ldr Q10, [x7, x8] + ldr Q11, [x7, x9] + tbl V7.16B, {V4.16B}, V10.16B + tbl V8.16B, {V5.16B}, V11.16B + str Q7, [x0] add x0, x0, x10, lsl 1 add x12, x12, x10 - str q8, [x0] + str Q8, [x0] add x0, x0, x11, lsl 1 add x12, x12, x11 subs w3, w3, #24 @@ -9195,13 +9195,13 @@ L_mlkem_rej_uniform_loop_lt_4 b L_mlkem_rej_uniform_loop_lt_4 L_mlkem_rej_uniform_done mov x0, x12 - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] ldp x29, x30, [sp], #0x40 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_sha3_aarch64_r DCQ 0x0000000000000001, 0x0000000000008082 @@ -9229,28 +9229,28 @@ mlkem_sha3_blocksx3_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x27, L_sha3_aarch64_r add x27, x27, L_sha3_aarch64_r str x0, [x29, #40] - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.d}[0], [x0] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.D}[0], [x0] add x0, x0, #8 - ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - ld1 {v24.d}[1], [x0] + ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + ld1 {V24.D}[1], [x0] add x0, x0, #8 ldp x1, x2, [x0] ldp x3, x4, [x0, #16] @@ -9270,154 +9270,154 @@ mlkem_sha3_blocksx3_neon PROC L_SHA3_transform_blocksx3_neon_begin stp x27, x28, [x29, #48] ; Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor3 V31.16B, V0.16B, V5.16B, V10.16B eor x0, x5, x10 - eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor3 V27.16B, V1.16B, V6.16B, V11.16B eor x30, x1, x6 - eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 V28.16B, V2.16B, V7.16B, V12.16B eor x28, x3, x8 - eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor3 V29.16B, V3.16B, V8.16B, V13.16B eor x0, x0, x15 - eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor3 V30.16B, V4.16B, V9.16B, V14.16B eor x30, x30, x11 - eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor3 V31.16B, V31.16B, V15.16B, V20.16B eor x28, x28, x13 - eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor3 V27.16B, V27.16B, V16.16B, V21.16B eor x0, x0, x21 - eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 V28.16B, V28.16B, V17.16B, V22.16B eor x30, x30, x16 - eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor3 V29.16B, V29.16B, V18.16B, V23.16B eor x28, x28, x19 - eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor3 V30.16B, V30.16B, V19.16B, V24.16B eor x0, x0, x26 - rax1 v25.2d, v30.2d, v27.2d + rax1 V25.2D, V30.2D, V27.2D eor x30, x30, x22 - rax1 v26.2d, v31.2d, v28.2d + rax1 V26.2D, V31.2D, V28.2D eor x28, x28, x24 - rax1 v27.2d, v27.2d, v29.2d + rax1 V27.2D, V27.2D, V29.2D str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d + rax1 V28.2D, V28.2D, V30.2D str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d + rax1 V29.2D, V29.2D, V31.2D eor x27, x2, x7 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 eor x28, x4, x9 - xar v1.2d, v6.2d, v26.2d, #20 + xar V1.2D, V6.2D, V26.2D, #20 eor x27, x27, x12 - xar v6.2d, v9.2d, v29.2d, #44 + xar V6.2D, V9.2D, V29.2D, #44 eor x28, x28, x14 - xar v9.2d, v22.2d, v27.2d, #3 + xar V9.2D, V22.2D, V27.2D, #3 eor x27, x27, x17 - xar v22.2d, v14.2d, v29.2d, #25 + xar V22.2D, V14.2D, V29.2D, #25 eor x28, x28, x20 - xar v14.2d, v20.2d, v25.2d, #46 + xar V14.2D, V20.2D, V25.2D, #46 eor x27, x27, x23 - xar v20.2d, v2.2d, v27.2d, #2 + xar V20.2D, V2.2D, V27.2D, #2 eor x28, x28, x25 - xar v2.2d, v12.2d, v27.2d, #21 + xar V2.2D, V12.2D, V27.2D, #21 eor x0, x0, x27, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 + xar V12.2D, V13.2D, V28.2D, #39 eor x27, x27, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 + xar V13.2D, V19.2D, V29.2D, #56 eor x1, x1, x0 - xar v19.2d, v23.2d, v28.2d, #8 + xar V19.2D, V23.2D, V28.2D, #8 eor x6, x6, x0 - xar v23.2d, v15.2d, v25.2d, #23 + xar V23.2D, V15.2D, V25.2D, #23 eor x11, x11, x0 - xar v15.2d, v4.2d, v29.2d, #37 + xar V15.2D, V4.2D, V29.2D, #37 eor x16, x16, x0 - xar v4.2d, v24.2d, v29.2d, #50 + xar V4.2D, V24.2D, V29.2D, #50 eor x22, x22, x0 - xar v24.2d, v21.2d, v26.2d, #62 + xar V24.2D, V21.2D, V26.2D, #62 eor x3, x3, x27 - xar v21.2d, v8.2d, v28.2d, #9 + xar V21.2D, V8.2D, V28.2D, #9 eor x8, x8, x27 - xar v8.2d, v16.2d, v26.2d, #19 + xar V8.2D, V16.2D, V26.2D, #19 eor x13, x13, x27 - xar v16.2d, v5.2d, v25.2d, #28 + xar V16.2D, V5.2D, V25.2D, #28 eor x19, x19, x27 - xar v5.2d, v3.2d, v28.2d, #36 + xar V5.2D, V3.2D, V28.2D, #36 eor x24, x24, x27 - xar v3.2d, v18.2d, v28.2d, #43 + xar V3.2D, V18.2D, V28.2D, #43 ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 + xar V18.2D, V17.2D, V27.2D, #49 ldr x27, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 + xar V17.2D, V11.2D, V26.2D, #54 eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 + xar V11.2D, V7.2D, V27.2D, #58 eor x30, x30, x27, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 + xar V7.2D, V10.2D, V25.2D, #61 eor x27, x27, x0, ror 63 ; Row Mix - mov v25.16b, v0.16b + mov V25.16B, V0.16B eor x5, x5, x28 - mov v26.16b, v1.16b + mov V26.16B, V1.16B eor x10, x10, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b + bcax V0.16B, V25.16B, V2.16B, V26.16B eor x15, x15, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b + bcax V1.16B, V26.16B, V3.16B, V2.16B eor x21, x21, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax V2.16B, V2.16B, V4.16B, V3.16B eor x26, x26, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b + bcax V3.16B, V3.16B, V25.16B, V4.16B eor x2, x2, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b + bcax V4.16B, V4.16B, V26.16B, V25.16B eor x7, x7, x30 - mov v25.16b, v5.16b + mov V25.16B, V5.16B eor x12, x12, x30 - mov v26.16b, v6.16b + mov V26.16B, V6.16B eor x17, x17, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b + bcax V5.16B, V25.16B, V7.16B, V26.16B eor x23, x23, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b + bcax V6.16B, V26.16B, V8.16B, V7.16B eor x4, x4, x27 - bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax V7.16B, V7.16B, V9.16B, V8.16B eor x9, x9, x27 - bcax v8.16b, v8.16b, v25.16b, v9.16b + bcax V8.16B, V8.16B, V25.16B, V9.16B eor x14, x14, x27 - bcax v9.16b, v9.16b, v26.16b, v25.16b + bcax V9.16B, V9.16B, V26.16B, V25.16B eor x20, x20, x27 - mov v26.16b, v11.16b + mov V26.16B, V11.16B eor x25, x25, x27 ; Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b + bcax V10.16B, V30.16B, V12.16B, V26.16B ror x0, x2, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b + bcax V11.16B, V26.16B, V13.16B, V12.16B ror x2, x7, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax V12.16B, V12.16B, V14.16B, V13.16B ror x7, x10, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b + bcax V13.16B, V13.16B, V30.16B, V14.16B ror x10, x24, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b + bcax V14.16B, V14.16B, V26.16B, V30.16B ror x24, x15, #25 - mov v25.16b, v15.16b + mov V25.16B, V15.16B ror x15, x22, #46 - mov v26.16b, v16.16b + mov V26.16B, V16.16B ror x22, x3, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b + bcax V15.16B, V25.16B, V17.16B, V26.16B ror x3, x13, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b + bcax V16.16B, V26.16B, V18.16B, V17.16B ror x13, x14, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax V17.16B, V17.16B, V19.16B, V18.16B ror x14, x21, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b + bcax V18.16B, V18.16B, V25.16B, V19.16B ror x21, x25, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b + bcax V19.16B, V19.16B, V26.16B, V25.16B ror x25, x16, #23 - mov v25.16b, v20.16b + mov V25.16B, V20.16B ror x16, x5, #37 - mov v26.16b, v21.16b + mov V26.16B, V21.16B ror x5, x26, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b + bcax V20.16B, V25.16B, V22.16B, V26.16B ror x26, x23, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b + bcax V21.16B, V26.16B, V23.16B, V22.16B ror x23, x9, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax V22.16B, V22.16B, V24.16B, V23.16B ror x9, x17, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b + bcax V23.16B, V23.16B, V25.16B, V24.16B ror x17, x6, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b + bcax V24.16B, V24.16B, V26.16B, V25.16B ror x6, x4, #36 ror x4, x20, #43 ror x20, x19, #49 @@ -9479,27 +9479,27 @@ L_SHA3_transform_blocksx3_neon_begin ldp x27, x28, [x29, #48] ldr x0, [x27], #8 subs x28, x28, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x1, x1, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_transform_blocksx3_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x1, x2, [x0] stp x3, x4, [x0, #16] @@ -9520,10 +9520,10 @@ L_SHA3_transform_blocksx3_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP @@ -9539,217 +9539,217 @@ mlkem_shake128_blocksx3_seed_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x28, L_sha3_aarch64_r add x28, x28, L_sha3_aarch64_r str x0, [x29, #40] add x0, x0, #32 - ld1 {v4.d}[0], [x0] + ld1 {V4.D}[0], [x0] ldp x2, x3, [x1], #16 add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] + ld1 {V4.D}[1], [x0] ldp x4, x5, [x1], #16 ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b + eor V5.16B, V5.16B, V5.16B eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b + eor V6.16B, V6.16B, V6.16B eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b + eor V8.16B, V8.16B, V8.16B eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b + eor V9.16B, V9.16B, V9.16B eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b + eor V10.16B, V10.16B, V10.16B eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b + eor V11.16B, V11.16B, V11.16B eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b + eor V12.16B, V12.16B, V12.16B eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b + eor V14.16B, V14.16B, V14.16B eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B eor x17, x17, x17 - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B eor x19, x19, x19 - eor v17.16b, v17.16b, v17.16b + eor V17.16B, V17.16B, V17.16B eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b + eor V18.16B, V18.16B, V18.16B eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b + eor V19.16B, V19.16B, V19.16B eor x22, x22, x22 movz x23, #0x8000, lsl 48 - eor v21.16b, v21.16b, v21.16b + eor V21.16B, V21.16B, V21.16B eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b + eor V22.16B, V22.16B, V22.16B eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b + eor V23.16B, V23.16B, V23.16B eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b + eor V24.16B, V24.16B, V24.16B eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v20.2d, x23 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V20.2D, x23 mov x1, #24 ; Start of 24 rounds L_SHA3_shake128_blocksx3_seed_neon_begin stp x28, x1, [x29, #48] ; Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor3 V31.16B, V0.16B, V5.16B, V10.16B eor x0, x6, x11 - eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor3 V27.16B, V1.16B, V6.16B, V11.16B eor x30, x2, x7 - eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 V28.16B, V2.16B, V7.16B, V12.16B eor x28, x4, x9 - eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor3 V29.16B, V3.16B, V8.16B, V13.16B eor x0, x0, x16 - eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor3 V30.16B, V4.16B, V9.16B, V14.16B eor x30, x30, x12 - eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor3 V31.16B, V31.16B, V15.16B, V20.16B eor x28, x28, x14 - eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor3 V27.16B, V27.16B, V16.16B, V21.16B eor x0, x0, x22 - eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 V28.16B, V28.16B, V17.16B, V22.16B eor x30, x30, x17 - eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor3 V29.16B, V29.16B, V18.16B, V23.16B eor x28, x28, x20 - eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor3 V30.16B, V30.16B, V19.16B, V24.16B eor x0, x0, x27 - rax1 v25.2d, v30.2d, v27.2d + rax1 V25.2D, V30.2D, V27.2D eor x30, x30, x23 - rax1 v26.2d, v31.2d, v28.2d + rax1 V26.2D, V31.2D, V28.2D eor x28, x28, x25 - rax1 v27.2d, v27.2d, v29.2d + rax1 V27.2D, V27.2D, V29.2D str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d + rax1 V28.2D, V28.2D, V30.2D str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d + rax1 V29.2D, V29.2D, V31.2D eor x1, x3, x8 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 eor x28, x5, x10 - xar v1.2d, v6.2d, v26.2d, #20 + xar V1.2D, V6.2D, V26.2D, #20 eor x1, x1, x13 - xar v6.2d, v9.2d, v29.2d, #44 + xar V6.2D, V9.2D, V29.2D, #44 eor x28, x28, x15 - xar v9.2d, v22.2d, v27.2d, #3 + xar V9.2D, V22.2D, V27.2D, #3 eor x1, x1, x19 - xar v22.2d, v14.2d, v29.2d, #25 + xar V22.2D, V14.2D, V29.2D, #25 eor x28, x28, x21 - xar v14.2d, v20.2d, v25.2d, #46 + xar V14.2D, V20.2D, V25.2D, #46 eor x1, x1, x24 - xar v20.2d, v2.2d, v27.2d, #2 + xar V20.2D, V2.2D, V27.2D, #2 eor x28, x28, x26 - xar v2.2d, v12.2d, v27.2d, #21 + xar V2.2D, V12.2D, V27.2D, #21 eor x0, x0, x1, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 + xar V12.2D, V13.2D, V28.2D, #39 eor x1, x1, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 + xar V13.2D, V19.2D, V29.2D, #56 eor x2, x2, x0 - xar v19.2d, v23.2d, v28.2d, #8 + xar V19.2D, V23.2D, V28.2D, #8 eor x7, x7, x0 - xar v23.2d, v15.2d, v25.2d, #23 + xar V23.2D, V15.2D, V25.2D, #23 eor x12, x12, x0 - xar v15.2d, v4.2d, v29.2d, #37 + xar V15.2D, V4.2D, V29.2D, #37 eor x17, x17, x0 - xar v4.2d, v24.2d, v29.2d, #50 + xar V4.2D, V24.2D, V29.2D, #50 eor x23, x23, x0 - xar v24.2d, v21.2d, v26.2d, #62 + xar V24.2D, V21.2D, V26.2D, #62 eor x4, x4, x1 - xar v21.2d, v8.2d, v28.2d, #9 + xar V21.2D, V8.2D, V28.2D, #9 eor x9, x9, x1 - xar v8.2d, v16.2d, v26.2d, #19 + xar V8.2D, V16.2D, V26.2D, #19 eor x14, x14, x1 - xar v16.2d, v5.2d, v25.2d, #28 + xar V16.2D, V5.2D, V25.2D, #28 eor x20, x20, x1 - xar v5.2d, v3.2d, v28.2d, #36 + xar V5.2D, V3.2D, V28.2D, #36 eor x25, x25, x1 - xar v3.2d, v18.2d, v28.2d, #43 + xar V3.2D, V18.2D, V28.2D, #43 ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 + xar V18.2D, V17.2D, V27.2D, #49 ldr x1, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 + xar V17.2D, V11.2D, V26.2D, #54 eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 + xar V11.2D, V7.2D, V27.2D, #58 eor x30, x30, x1, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 + xar V7.2D, V10.2D, V25.2D, #61 eor x1, x1, x0, ror 63 ; Row Mix - mov v25.16b, v0.16b + mov V25.16B, V0.16B eor x6, x6, x28 - mov v26.16b, v1.16b + mov V26.16B, V1.16B eor x11, x11, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b + bcax V0.16B, V25.16B, V2.16B, V26.16B eor x16, x16, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b + bcax V1.16B, V26.16B, V3.16B, V2.16B eor x22, x22, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax V2.16B, V2.16B, V4.16B, V3.16B eor x27, x27, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b + bcax V3.16B, V3.16B, V25.16B, V4.16B eor x3, x3, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b + bcax V4.16B, V4.16B, V26.16B, V25.16B eor x8, x8, x30 - mov v25.16b, v5.16b + mov V25.16B, V5.16B eor x13, x13, x30 - mov v26.16b, v6.16b + mov V26.16B, V6.16B eor x19, x19, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b + bcax V5.16B, V25.16B, V7.16B, V26.16B eor x24, x24, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b + bcax V6.16B, V26.16B, V8.16B, V7.16B eor x5, x5, x1 - bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax V7.16B, V7.16B, V9.16B, V8.16B eor x10, x10, x1 - bcax v8.16b, v8.16b, v25.16b, v9.16b + bcax V8.16B, V8.16B, V25.16B, V9.16B eor x15, x15, x1 - bcax v9.16b, v9.16b, v26.16b, v25.16b + bcax V9.16B, V9.16B, V26.16B, V25.16B eor x21, x21, x1 - mov v26.16b, v11.16b + mov V26.16B, V11.16B eor x26, x26, x1 ; Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b + bcax V10.16B, V30.16B, V12.16B, V26.16B ror x0, x3, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b + bcax V11.16B, V26.16B, V13.16B, V12.16B ror x3, x8, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax V12.16B, V12.16B, V14.16B, V13.16B ror x8, x11, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b + bcax V13.16B, V13.16B, V30.16B, V14.16B ror x11, x25, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b + bcax V14.16B, V14.16B, V26.16B, V30.16B ror x25, x16, #25 - mov v25.16b, v15.16b + mov V25.16B, V15.16B ror x16, x23, #46 - mov v26.16b, v16.16b + mov V26.16B, V16.16B ror x23, x4, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b + bcax V15.16B, V25.16B, V17.16B, V26.16B ror x4, x14, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b + bcax V16.16B, V26.16B, V18.16B, V17.16B ror x14, x15, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax V17.16B, V17.16B, V19.16B, V18.16B ror x15, x22, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b + bcax V18.16B, V18.16B, V25.16B, V19.16B ror x22, x26, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b + bcax V19.16B, V19.16B, V26.16B, V25.16B ror x26, x17, #23 - mov v25.16b, v20.16b + mov V25.16B, V20.16B ror x17, x6, #37 - mov v26.16b, v21.16b + mov V26.16B, V21.16B ror x6, x27, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b + bcax V20.16B, V25.16B, V22.16B, V26.16B ror x27, x24, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b + bcax V21.16B, V26.16B, V23.16B, V22.16B ror x24, x10, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax V22.16B, V22.16B, V24.16B, V23.16B ror x10, x19, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b + bcax V23.16B, V23.16B, V25.16B, V24.16B ror x19, x7, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b + bcax V24.16B, V24.16B, V26.16B, V25.16B ror x7, x5, #36 ror x5, x21, #43 ror x21, x20, #49 @@ -9811,27 +9811,27 @@ L_SHA3_shake128_blocksx3_seed_neon_begin ldp x28, x1, [x29, #48] ldr x0, [x28], #8 subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_shake128_blocksx3_seed_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x2, x3, [x0] stp x4, x5, [x0, #16] @@ -9852,10 +9852,10 @@ L_SHA3_shake128_blocksx3_seed_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP @@ -9871,217 +9871,217 @@ mlkem_shake256_blocksx3_seed_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x28, L_sha3_aarch64_r add x28, x28, L_sha3_aarch64_r str x0, [x29, #40] add x0, x0, #32 - ld1 {v4.d}[0], [x0] + ld1 {V4.D}[0], [x0] ldp x2, x3, [x1], #16 add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] + ld1 {V4.D}[1], [x0] ldp x4, x5, [x1], #16 ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b + eor V5.16B, V5.16B, V5.16B eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b + eor V6.16B, V6.16B, V6.16B eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b + eor V8.16B, V8.16B, V8.16B eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b + eor V9.16B, V9.16B, V9.16B eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b + eor V10.16B, V10.16B, V10.16B eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b + eor V11.16B, V11.16B, V11.16B eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b + eor V12.16B, V12.16B, V12.16B eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b + eor V14.16B, V14.16B, V14.16B eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B eor x17, x17, x17 movz x19, #0x8000, lsl 48 - eor v17.16b, v17.16b, v17.16b + eor V17.16B, V17.16B, V17.16B eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b + eor V18.16B, V18.16B, V18.16B eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b + eor V19.16B, V19.16B, V19.16B eor x22, x22, x22 - eor v20.16b, v20.16b, v20.16b + eor V20.16B, V20.16B, V20.16B eor x23, x23, x23 - eor v21.16b, v21.16b, v21.16b + eor V21.16B, V21.16B, V21.16B eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b + eor V22.16B, V22.16B, V22.16B eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b + eor V23.16B, V23.16B, V23.16B eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b + eor V24.16B, V24.16B, V24.16B eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v16.2d, x19 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V16.2D, x19 mov x1, #24 ; Start of 24 rounds L_SHA3_shake256_blocksx3_seed_neon_begin stp x28, x1, [x29, #48] ; Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor3 V31.16B, V0.16B, V5.16B, V10.16B eor x0, x6, x11 - eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor3 V27.16B, V1.16B, V6.16B, V11.16B eor x30, x2, x7 - eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor3 V28.16B, V2.16B, V7.16B, V12.16B eor x28, x4, x9 - eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor3 V29.16B, V3.16B, V8.16B, V13.16B eor x0, x0, x16 - eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor3 V30.16B, V4.16B, V9.16B, V14.16B eor x30, x30, x12 - eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor3 V31.16B, V31.16B, V15.16B, V20.16B eor x28, x28, x14 - eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor3 V27.16B, V27.16B, V16.16B, V21.16B eor x0, x0, x22 - eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor3 V28.16B, V28.16B, V17.16B, V22.16B eor x30, x30, x17 - eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor3 V29.16B, V29.16B, V18.16B, V23.16B eor x28, x28, x20 - eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor3 V30.16B, V30.16B, V19.16B, V24.16B eor x0, x0, x27 - rax1 v25.2d, v30.2d, v27.2d + rax1 V25.2D, V30.2D, V27.2D eor x30, x30, x23 - rax1 v26.2d, v31.2d, v28.2d + rax1 V26.2D, V31.2D, V28.2D eor x28, x28, x25 - rax1 v27.2d, v27.2d, v29.2d + rax1 V27.2D, V27.2D, V29.2D str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d + rax1 V28.2D, V28.2D, V30.2D str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d + rax1 V29.2D, V29.2D, V31.2D eor x1, x3, x8 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 eor x28, x5, x10 - xar v1.2d, v6.2d, v26.2d, #20 + xar V1.2D, V6.2D, V26.2D, #20 eor x1, x1, x13 - xar v6.2d, v9.2d, v29.2d, #44 + xar V6.2D, V9.2D, V29.2D, #44 eor x28, x28, x15 - xar v9.2d, v22.2d, v27.2d, #3 + xar V9.2D, V22.2D, V27.2D, #3 eor x1, x1, x19 - xar v22.2d, v14.2d, v29.2d, #25 + xar V22.2D, V14.2D, V29.2D, #25 eor x28, x28, x21 - xar v14.2d, v20.2d, v25.2d, #46 + xar V14.2D, V20.2D, V25.2D, #46 eor x1, x1, x24 - xar v20.2d, v2.2d, v27.2d, #2 + xar V20.2D, V2.2D, V27.2D, #2 eor x28, x28, x26 - xar v2.2d, v12.2d, v27.2d, #21 + xar V2.2D, V12.2D, V27.2D, #21 eor x0, x0, x1, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 + xar V12.2D, V13.2D, V28.2D, #39 eor x1, x1, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 + xar V13.2D, V19.2D, V29.2D, #56 eor x2, x2, x0 - xar v19.2d, v23.2d, v28.2d, #8 + xar V19.2D, V23.2D, V28.2D, #8 eor x7, x7, x0 - xar v23.2d, v15.2d, v25.2d, #23 + xar V23.2D, V15.2D, V25.2D, #23 eor x12, x12, x0 - xar v15.2d, v4.2d, v29.2d, #37 + xar V15.2D, V4.2D, V29.2D, #37 eor x17, x17, x0 - xar v4.2d, v24.2d, v29.2d, #50 + xar V4.2D, V24.2D, V29.2D, #50 eor x23, x23, x0 - xar v24.2d, v21.2d, v26.2d, #62 + xar V24.2D, V21.2D, V26.2D, #62 eor x4, x4, x1 - xar v21.2d, v8.2d, v28.2d, #9 + xar V21.2D, V8.2D, V28.2D, #9 eor x9, x9, x1 - xar v8.2d, v16.2d, v26.2d, #19 + xar V8.2D, V16.2D, V26.2D, #19 eor x14, x14, x1 - xar v16.2d, v5.2d, v25.2d, #28 + xar V16.2D, V5.2D, V25.2D, #28 eor x20, x20, x1 - xar v5.2d, v3.2d, v28.2d, #36 + xar V5.2D, V3.2D, V28.2D, #36 eor x25, x25, x1 - xar v3.2d, v18.2d, v28.2d, #43 + xar V3.2D, V18.2D, V28.2D, #43 ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 + xar V18.2D, V17.2D, V27.2D, #49 ldr x1, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 + xar V17.2D, V11.2D, V26.2D, #54 eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 + xar V11.2D, V7.2D, V27.2D, #58 eor x30, x30, x1, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 + xar V7.2D, V10.2D, V25.2D, #61 eor x1, x1, x0, ror 63 ; Row Mix - mov v25.16b, v0.16b + mov V25.16B, V0.16B eor x6, x6, x28 - mov v26.16b, v1.16b + mov V26.16B, V1.16B eor x11, x11, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b + bcax V0.16B, V25.16B, V2.16B, V26.16B eor x16, x16, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b + bcax V1.16B, V26.16B, V3.16B, V2.16B eor x22, x22, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b + bcax V2.16B, V2.16B, V4.16B, V3.16B eor x27, x27, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b + bcax V3.16B, V3.16B, V25.16B, V4.16B eor x3, x3, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b + bcax V4.16B, V4.16B, V26.16B, V25.16B eor x8, x8, x30 - mov v25.16b, v5.16b + mov V25.16B, V5.16B eor x13, x13, x30 - mov v26.16b, v6.16b + mov V26.16B, V6.16B eor x19, x19, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b + bcax V5.16B, V25.16B, V7.16B, V26.16B eor x24, x24, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b + bcax V6.16B, V26.16B, V8.16B, V7.16B eor x5, x5, x1 - bcax v7.16b, v7.16b, v9.16b, v8.16b + bcax V7.16B, V7.16B, V9.16B, V8.16B eor x10, x10, x1 - bcax v8.16b, v8.16b, v25.16b, v9.16b + bcax V8.16B, V8.16B, V25.16B, V9.16B eor x15, x15, x1 - bcax v9.16b, v9.16b, v26.16b, v25.16b + bcax V9.16B, V9.16B, V26.16B, V25.16B eor x21, x21, x1 - mov v26.16b, v11.16b + mov V26.16B, V11.16B eor x26, x26, x1 ; Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b + bcax V10.16B, V30.16B, V12.16B, V26.16B ror x0, x3, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b + bcax V11.16B, V26.16B, V13.16B, V12.16B ror x3, x8, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b + bcax V12.16B, V12.16B, V14.16B, V13.16B ror x8, x11, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b + bcax V13.16B, V13.16B, V30.16B, V14.16B ror x11, x25, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b + bcax V14.16B, V14.16B, V26.16B, V30.16B ror x25, x16, #25 - mov v25.16b, v15.16b + mov V25.16B, V15.16B ror x16, x23, #46 - mov v26.16b, v16.16b + mov V26.16B, V16.16B ror x23, x4, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b + bcax V15.16B, V25.16B, V17.16B, V26.16B ror x4, x14, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b + bcax V16.16B, V26.16B, V18.16B, V17.16B ror x14, x15, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b + bcax V17.16B, V17.16B, V19.16B, V18.16B ror x15, x22, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b + bcax V18.16B, V18.16B, V25.16B, V19.16B ror x22, x26, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b + bcax V19.16B, V19.16B, V26.16B, V25.16B ror x26, x17, #23 - mov v25.16b, v20.16b + mov V25.16B, V20.16B ror x17, x6, #37 - mov v26.16b, v21.16b + mov V26.16B, V21.16B ror x6, x27, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b + bcax V20.16B, V25.16B, V22.16B, V26.16B ror x27, x24, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b + bcax V21.16B, V26.16B, V23.16B, V22.16B ror x24, x10, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b + bcax V22.16B, V22.16B, V24.16B, V23.16B ror x10, x19, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b + bcax V23.16B, V23.16B, V25.16B, V24.16B ror x19, x7, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b + bcax V24.16B, V24.16B, V26.16B, V25.16B ror x7, x5, #36 ror x5, x21, #43 ror x21, x20, #49 @@ -10143,27 +10143,27 @@ L_SHA3_shake256_blocksx3_seed_neon_begin ldp x28, x1, [x29, #48] ldr x0, [x28], #8 subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_shake256_blocksx3_seed_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x2, x3, [x0] stp x4, x5, [x0, #16] @@ -10184,10 +10184,10 @@ L_SHA3_shake256_blocksx3_seed_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP @@ -10204,28 +10204,28 @@ mlkem_sha3_blocksx3_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x27, L_sha3_aarch64_r add x27, x27, L_sha3_aarch64_r str x0, [x29, #40] - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.d}[0], [x0] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.D}[0], [x0] add x0, x0, #8 - ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - ld1 {v24.d}[1], [x0] + ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + ld1 {V24.D}[1], [x0] add x0, x0, #8 ldp x1, x2, [x0] ldp x3, x4, [x0, #16] @@ -10245,321 +10245,321 @@ mlkem_sha3_blocksx3_neon PROC L_SHA3_transform_blocksx3_neon_begin stp x27, x28, [x29, #48] ; Col Mix NEON - eor v30.16b, v4.16b, v9.16b + eor V30.16B, V4.16B, V9.16B eor x0, x5, x10 - eor v27.16b, v1.16b, v6.16b + eor V27.16B, V1.16B, V6.16B eor x30, x1, x6 - eor v30.16b, v30.16b, v14.16b + eor V30.16B, V30.16B, V14.16B eor x28, x3, x8 - eor v27.16b, v27.16b, v11.16b + eor V27.16B, V27.16B, V11.16B eor x0, x0, x15 - eor v30.16b, v30.16b, v19.16b + eor V30.16B, V30.16B, V19.16B eor x30, x30, x11 - eor v27.16b, v27.16b, v16.16b + eor V27.16B, V27.16B, V16.16B eor x28, x28, x13 - eor v30.16b, v30.16b, v24.16b + eor V30.16B, V30.16B, V24.16B eor x0, x0, x21 - eor v27.16b, v27.16b, v21.16b + eor V27.16B, V27.16B, V21.16B eor x30, x30, x16 - ushr v25.2d, v27.2d, #63 + ushr V25.2D, V27.2D, #63 eor x28, x28, x19 - sli v25.2d, v27.2d, #1 + sli V25.2D, V27.2D, #1 eor x0, x0, x26 - eor v25.16b, v25.16b, v30.16b + eor V25.16B, V25.16B, V30.16B eor x30, x30, x22 - eor v31.16b, v0.16b, v5.16b + eor V31.16B, V0.16B, V5.16B eor x28, x28, x24 - eor v28.16b, v2.16b, v7.16b + eor V28.16B, V2.16B, V7.16B str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b + eor V31.16B, V31.16B, V10.16B str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b + eor V28.16B, V28.16B, V12.16B eor x27, x2, x7 - eor v31.16b, v31.16b, v15.16b + eor V31.16B, V31.16B, V15.16B eor x28, x4, x9 - eor v28.16b, v28.16b, v17.16b + eor V28.16B, V28.16B, V17.16B eor x27, x27, x12 - eor v31.16b, v31.16b, v20.16b + eor V31.16B, V31.16B, V20.16B eor x28, x28, x14 - eor v28.16b, v28.16b, v22.16b + eor V28.16B, V28.16B, V22.16B eor x27, x27, x17 - ushr v29.2d, v30.2d, #63 + ushr V29.2D, V30.2D, #63 eor x28, x28, x20 - ushr v26.2d, v28.2d, #63 + ushr V26.2D, V28.2D, #63 eor x27, x27, x23 - sli v29.2d, v30.2d, #1 + sli V29.2D, V30.2D, #1 eor x28, x28, x25 - sli v26.2d, v28.2d, #1 + sli V26.2D, V28.2D, #1 eor x0, x0, x27, ror 63 - eor v28.16b, v28.16b, v29.16b + eor V28.16B, V28.16B, V29.16B eor x27, x27, x28, ror 63 - eor v29.16b, v3.16b, v8.16b + eor V29.16B, V3.16B, V8.16B eor x1, x1, x0 - eor v26.16b, v26.16b, v31.16b + eor V26.16B, V26.16B, V31.16B eor x6, x6, x0 - eor v29.16b, v29.16b, v13.16b + eor V29.16B, V29.16B, V13.16B eor x11, x11, x0 - eor v29.16b, v29.16b, v18.16b + eor V29.16B, V29.16B, V18.16B eor x16, x16, x0 - eor v29.16b, v29.16b, v23.16b + eor V29.16B, V29.16B, V23.16B eor x22, x22, x0 - ushr v30.2d, v29.2d, #63 + ushr V30.2D, V29.2D, #63 eor x3, x3, x27 - sli v30.2d, v29.2d, #1 + sli V30.2D, V29.2D, #1 eor x8, x8, x27 - eor v27.16b, v27.16b, v30.16b + eor V27.16B, V27.16B, V30.16B eor x13, x13, x27 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x19, x19, x27 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x24, x24, x27 - eor v29.16b, v29.16b, v30.16b + eor V29.16B, V29.16B, V30.16B ldr x0, [x29, #32] ; Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B ldr x27, [x29, #24] - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x30, x30, x27, ror 63 - ushr v1.2d, v6.2d, #20 + ushr V1.2D, V6.2D, #20 eor x27, x27, x0, ror 63 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x5, x5, x28 - sli v1.2d, v6.2d, #44 + sli V1.2D, V6.2D, #44 eor x10, x10, x28 - eor v31.16b, v9.16b, v29.16b + eor V31.16B, V9.16B, V29.16B eor x15, x15, x28 - eor v22.16b, v22.16b, v27.16b + eor V22.16B, V22.16B, V27.16B eor x21, x21, x28 - ushr v6.2d, v31.2d, #44 + ushr V6.2D, V31.2D, #44 eor x26, x26, x28 - ushr v9.2d, v22.2d, #3 + ushr V9.2D, V22.2D, #3 eor x2, x2, x30 - sli v6.2d, v31.2d, #20 + sli V6.2D, V31.2D, #20 eor x7, x7, x30 - sli v9.2d, v22.2d, #61 + sli V9.2D, V22.2D, #61 eor x12, x12, x30 - eor v31.16b, v14.16b, v29.16b + eor V31.16B, V14.16B, V29.16B eor x17, x17, x30 - eor v20.16b, v20.16b, v25.16b + eor V20.16B, V20.16B, V25.16B eor x23, x23, x30 - ushr v22.2d, v31.2d, #25 + ushr V22.2D, V31.2D, #25 eor x4, x4, x27 - ushr v14.2d, v20.2d, #46 + ushr V14.2D, V20.2D, #46 eor x9, x9, x27 - sli v22.2d, v31.2d, #39 + sli V22.2D, V31.2D, #39 eor x14, x14, x27 - sli v14.2d, v20.2d, #18 + sli V14.2D, V20.2D, #18 eor x20, x20, x27 - eor v31.16b, v2.16b, v27.16b + eor V31.16B, V2.16B, V27.16B eor x25, x25, x27 ; Swap Rotate Base - eor v12.16b, v12.16b, v27.16b + eor V12.16B, V12.16B, V27.16B ror x0, x2, #63 - ushr v20.2d, v31.2d, #2 + ushr V20.2D, V31.2D, #2 ror x2, x7, #20 - ushr v2.2d, v12.2d, #21 + ushr V2.2D, V12.2D, #21 ror x7, x10, #44 - sli v20.2d, v31.2d, #62 + sli V20.2D, V31.2D, #62 ror x10, x24, #3 - sli v2.2d, v12.2d, #43 + sli V2.2D, V12.2D, #43 ror x24, x15, #25 - eor v31.16b, v13.16b, v28.16b + eor V31.16B, V13.16B, V28.16B ror x15, x22, #46 - eor v19.16b, v19.16b, v29.16b + eor V19.16B, V19.16B, V29.16B ror x22, x3, #2 - ushr v12.2d, v31.2d, #39 + ushr V12.2D, V31.2D, #39 ror x3, x13, #21 - ushr v13.2d, v19.2d, #56 + ushr V13.2D, V19.2D, #56 ror x13, x14, #39 - sli v12.2d, v31.2d, #25 + sli V12.2D, V31.2D, #25 ror x14, x21, #56 - sli v13.2d, v19.2d, #8 + sli V13.2D, V19.2D, #8 ror x21, x25, #8 - eor v31.16b, v23.16b, v28.16b + eor V31.16B, V23.16B, V28.16B ror x25, x16, #23 - eor v15.16b, v15.16b, v25.16b + eor V15.16B, V15.16B, V25.16B ror x16, x5, #37 - ushr v19.2d, v31.2d, #8 + ushr V19.2D, V31.2D, #8 ror x5, x26, #50 - ushr v23.2d, v15.2d, #23 + ushr V23.2D, V15.2D, #23 ror x26, x23, #62 - sli v19.2d, v31.2d, #56 + sli V19.2D, V31.2D, #56 ror x23, x9, #9 - sli v23.2d, v15.2d, #41 + sli V23.2D, V15.2D, #41 ror x9, x17, #19 - eor v31.16b, v4.16b, v29.16b + eor V31.16B, V4.16B, V29.16B ror x17, x6, #28 - eor v24.16b, v24.16b, v29.16b + eor V24.16B, V24.16B, V29.16B ror x6, x4, #36 - ushr v15.2d, v31.2d, #37 + ushr V15.2D, V31.2D, #37 ror x4, x20, #43 - ushr v4.2d, v24.2d, #50 + ushr V4.2D, V24.2D, #50 ror x20, x19, #49 - sli v15.2d, v31.2d, #27 + sli V15.2D, V31.2D, #27 ror x19, x12, #54 - sli v4.2d, v24.2d, #14 + sli V4.2D, V24.2D, #14 ror x12, x8, #58 - eor v31.16b, v21.16b, v26.16b + eor V31.16B, V21.16B, V26.16B ror x8, x11, #61 ; Row Mix Base - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B bic x11, x3, x2 - ushr v24.2d, v31.2d, #62 + ushr V24.2D, V31.2D, #62 bic x27, x4, x3 - ushr v21.2d, v8.2d, #9 + ushr V21.2D, V8.2D, #9 bic x28, x1, x5 - sli v24.2d, v31.2d, #2 + sli V24.2D, V31.2D, #2 bic x30, x2, x1 - sli v21.2d, v8.2d, #55 + sli V21.2D, V8.2D, #55 eor x1, x1, x11 - eor v31.16b, v16.16b, v26.16b + eor V31.16B, V16.16B, V26.16B eor x2, x2, x27 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B bic x11, x5, x4 - ushr v8.2d, v31.2d, #19 + ushr V8.2D, V31.2D, #19 eor x4, x4, x28 - ushr v16.2d, v5.2d, #28 + ushr V16.2D, V5.2D, #28 eor x3, x3, x11 - sli v8.2d, v31.2d, #45 + sli V8.2D, V31.2D, #45 eor x5, x5, x30 - sli v16.2d, v5.2d, #36 + sli V16.2D, V5.2D, #36 bic x11, x8, x7 - eor v31.16b, v3.16b, v28.16b + eor V31.16B, V3.16B, V28.16B bic x27, x9, x8 - eor v18.16b, v18.16b, v28.16b + eor V18.16B, V18.16B, V28.16B bic x28, x6, x10 - ushr v5.2d, v31.2d, #36 + ushr V5.2D, V31.2D, #36 bic x30, x7, x6 - ushr v3.2d, v18.2d, #43 + ushr V3.2D, V18.2D, #43 eor x6, x6, x11 - sli v5.2d, v31.2d, #28 + sli V5.2D, V31.2D, #28 eor x7, x7, x27 - sli v3.2d, v18.2d, #21 + sli V3.2D, V18.2D, #21 bic x11, x10, x9 - eor v31.16b, v17.16b, v27.16b + eor V31.16B, V17.16B, V27.16B eor x9, x9, x28 - eor v11.16b, v11.16b, v26.16b + eor V11.16B, V11.16B, V26.16B eor x8, x8, x11 - ushr v18.2d, v31.2d, #49 + ushr V18.2D, V31.2D, #49 eor x10, x10, x30 - ushr v17.2d, v11.2d, #54 + ushr V17.2D, V11.2D, #54 bic x11, x13, x12 - sli v18.2d, v31.2d, #15 + sli V18.2D, V31.2D, #15 bic x27, x14, x13 - sli v17.2d, v11.2d, #10 + sli V17.2D, V11.2D, #10 bic x28, x0, x15 - eor v31.16b, v7.16b, v27.16b + eor V31.16B, V7.16B, V27.16B bic x30, x12, x0 - eor v10.16b, v10.16b, v25.16b + eor V10.16B, V10.16B, V25.16B eor x11, x0, x11 - ushr v11.2d, v31.2d, #58 + ushr V11.2D, V31.2D, #58 eor x12, x12, x27 - ushr v7.2d, v10.2d, #61 + ushr V7.2D, V10.2D, #61 bic x0, x15, x14 - sli v11.2d, v31.2d, #6 + sli V11.2D, V31.2D, #6 eor x14, x14, x28 - sli v7.2d, v10.2d, #3 + sli V7.2D, V10.2D, #3 eor x13, x13, x0 ; Row Mix NEON - bic v25.16b, v2.16b, v1.16b + bic V25.16B, V2.16B, V1.16B eor x15, x15, x30 - bic v26.16b, v3.16b, v2.16b + bic V26.16B, V3.16B, V2.16B bic x0, x19, x17 - bic v27.16b, v4.16b, v3.16b + bic V27.16B, V4.16B, V3.16B bic x27, x20, x19 - bic v28.16b, v0.16b, v4.16b + bic V28.16B, V0.16B, V4.16B bic x28, x16, x21 - bic v29.16b, v1.16b, v0.16b + bic V29.16B, V1.16B, V0.16B bic x30, x17, x16 - eor v0.16b, v0.16b, v25.16b + eor V0.16B, V0.16B, V25.16B eor x16, x16, x0 - eor v1.16b, v1.16b, v26.16b + eor V1.16B, V1.16B, V26.16B eor x17, x17, x27 - eor v2.16b, v2.16b, v27.16b + eor V2.16B, V2.16B, V27.16B bic x0, x21, x20 - eor v3.16b, v3.16b, v28.16b + eor V3.16B, V3.16B, V28.16B eor x20, x20, x28 - eor v4.16b, v4.16b, v29.16b + eor V4.16B, V4.16B, V29.16B eor x19, x19, x0 - bic v25.16b, v7.16b, v6.16b + bic V25.16B, V7.16B, V6.16B eor x21, x21, x30 - bic v26.16b, v8.16b, v7.16b + bic V26.16B, V8.16B, V7.16B bic x0, x24, x23 - bic v27.16b, v9.16b, v8.16b + bic V27.16B, V9.16B, V8.16B bic x27, x25, x24 - bic v28.16b, v5.16b, v9.16b + bic V28.16B, V5.16B, V9.16B bic x28, x22, x26 - bic v29.16b, v6.16b, v5.16b + bic V29.16B, V6.16B, V5.16B bic x30, x23, x22 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B eor x22, x22, x0 - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x23, x23, x27 - eor v7.16b, v7.16b, v27.16b + eor V7.16B, V7.16B, V27.16B bic x0, x26, x25 - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B eor x25, x25, x28 - eor v9.16b, v9.16b, v29.16b + eor V9.16B, V9.16B, V29.16B eor x24, x24, x0 - bic v25.16b, v12.16b, v11.16b + bic V25.16B, V12.16B, V11.16B eor x26, x26, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B ; Done transforming ldp x27, x28, [x29, #48] ldr x0, [x27], #8 subs x28, x28, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x1, x1, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_transform_blocksx3_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x1, x2, [x0] stp x3, x4, [x0, #16] @@ -10580,10 +10580,10 @@ L_SHA3_transform_blocksx3_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP @@ -10599,384 +10599,384 @@ mlkem_shake128_blocksx3_seed_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x28, L_sha3_aarch64_r add x28, x28, L_sha3_aarch64_r str x0, [x29, #40] add x0, x0, #32 - ld1 {v4.d}[0], [x0] + ld1 {V4.D}[0], [x0] ldp x2, x3, [x1], #16 add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] + ld1 {V4.D}[1], [x0] ldp x4, x5, [x1], #16 ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b + eor V5.16B, V5.16B, V5.16B eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b + eor V6.16B, V6.16B, V6.16B eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b + eor V8.16B, V8.16B, V8.16B eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b + eor V9.16B, V9.16B, V9.16B eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b + eor V10.16B, V10.16B, V10.16B eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b + eor V11.16B, V11.16B, V11.16B eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b + eor V12.16B, V12.16B, V12.16B eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b + eor V14.16B, V14.16B, V14.16B eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B eor x17, x17, x17 - eor v16.16b, v16.16b, v16.16b + eor V16.16B, V16.16B, V16.16B eor x19, x19, x19 - eor v17.16b, v17.16b, v17.16b + eor V17.16B, V17.16B, V17.16B eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b + eor V18.16B, V18.16B, V18.16B eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b + eor V19.16B, V19.16B, V19.16B eor x22, x22, x22 movz x23, #0x8000, lsl 48 - eor v21.16b, v21.16b, v21.16b + eor V21.16B, V21.16B, V21.16B eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b + eor V22.16B, V22.16B, V22.16B eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b + eor V23.16B, V23.16B, V23.16B eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b + eor V24.16B, V24.16B, V24.16B eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v20.2d, x23 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V20.2D, x23 mov x1, #24 ; Start of 24 rounds L_SHA3_shake128_blocksx3_seed_neon_begin stp x28, x1, [x29, #48] ; Col Mix NEON - eor v30.16b, v4.16b, v9.16b + eor V30.16B, V4.16B, V9.16B eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b + eor V27.16B, V1.16B, V6.16B eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b + eor V30.16B, V30.16B, V14.16B eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b + eor V27.16B, V27.16B, V11.16B eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b + eor V30.16B, V30.16B, V19.16B eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b + eor V27.16B, V27.16B, V16.16B eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b + eor V30.16B, V30.16B, V24.16B eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b + eor V27.16B, V27.16B, V21.16B eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 + ushr V25.2D, V27.2D, #63 eor x28, x28, x20 - sli v25.2d, v27.2d, #1 + sli V25.2D, V27.2D, #1 eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b + eor V25.16B, V25.16B, V30.16B eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b + eor V31.16B, V0.16B, V5.16B eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b + eor V28.16B, V2.16B, V7.16B str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b + eor V31.16B, V31.16B, V10.16B str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b + eor V28.16B, V28.16B, V12.16B eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b + eor V31.16B, V31.16B, V15.16B eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b + eor V28.16B, V28.16B, V17.16B eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b + eor V31.16B, V31.16B, V20.16B eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b + eor V28.16B, V28.16B, V22.16B eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 + ushr V29.2D, V30.2D, #63 eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 + ushr V26.2D, V28.2D, #63 eor x1, x1, x24 - sli v29.2d, v30.2d, #1 + sli V29.2D, V30.2D, #1 eor x28, x28, x26 - sli v26.2d, v28.2d, #1 + sli V26.2D, V28.2D, #1 eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b + eor V28.16B, V28.16B, V29.16B eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b + eor V29.16B, V3.16B, V8.16B eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b + eor V26.16B, V26.16B, V31.16B eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b + eor V29.16B, V29.16B, V13.16B eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b + eor V29.16B, V29.16B, V18.16B eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b + eor V29.16B, V29.16B, V23.16B eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 + ushr V30.2D, V29.2D, #63 eor x4, x4, x1 - sli v30.2d, v29.2d, #1 + sli V30.2D, V29.2D, #1 eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b + eor V27.16B, V27.16B, V30.16B eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x20, x20, x1 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b + eor V29.16B, V29.16B, V30.16B ldr x0, [x29, #32] ; Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 + ushr V1.2D, V6.2D, #20 eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x6, x6, x28 - sli v1.2d, v6.2d, #44 + sli V1.2D, V6.2D, #44 eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b + eor V31.16B, V9.16B, V29.16B eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b + eor V22.16B, V22.16B, V27.16B eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 + ushr V6.2D, V31.2D, #44 eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 + ushr V9.2D, V22.2D, #3 eor x3, x3, x30 - sli v6.2d, v31.2d, #20 + sli V6.2D, V31.2D, #20 eor x8, x8, x30 - sli v9.2d, v22.2d, #61 + sli V9.2D, V22.2D, #61 eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b + eor V31.16B, V14.16B, V29.16B eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b + eor V20.16B, V20.16B, V25.16B eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 + ushr V22.2D, V31.2D, #25 eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 + ushr V14.2D, V20.2D, #46 eor x10, x10, x1 - sli v22.2d, v31.2d, #39 + sli V22.2D, V31.2D, #39 eor x15, x15, x1 - sli v14.2d, v20.2d, #18 + sli V14.2D, V20.2D, #18 eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b + eor V31.16B, V2.16B, V27.16B eor x26, x26, x1 ; Swap Rotate Base - eor v12.16b, v12.16b, v27.16b + eor V12.16B, V12.16B, V27.16B ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 + ushr V20.2D, V31.2D, #2 ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 + ushr V2.2D, V12.2D, #21 ror x8, x11, #44 - sli v20.2d, v31.2d, #62 + sli V20.2D, V31.2D, #62 ror x11, x25, #3 - sli v2.2d, v12.2d, #43 + sli V2.2D, V12.2D, #43 ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b + eor V31.16B, V13.16B, V28.16B ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b + eor V19.16B, V19.16B, V29.16B ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 + ushr V12.2D, V31.2D, #39 ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 + ushr V13.2D, V19.2D, #56 ror x14, x15, #39 - sli v12.2d, v31.2d, #25 + sli V12.2D, V31.2D, #25 ror x15, x22, #56 - sli v13.2d, v19.2d, #8 + sli V13.2D, V19.2D, #8 ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b + eor V31.16B, V23.16B, V28.16B ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b + eor V15.16B, V15.16B, V25.16B ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 + ushr V19.2D, V31.2D, #8 ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 + ushr V23.2D, V15.2D, #23 ror x27, x24, #62 - sli v19.2d, v31.2d, #56 + sli V19.2D, V31.2D, #56 ror x24, x10, #9 - sli v23.2d, v15.2d, #41 + sli V23.2D, V15.2D, #41 ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b + eor V31.16B, V4.16B, V29.16B ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b + eor V24.16B, V24.16B, V29.16B ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 + ushr V15.2D, V31.2D, #37 ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 + ushr V4.2D, V24.2D, #50 ror x21, x20, #49 - sli v15.2d, v31.2d, #27 + sli V15.2D, V31.2D, #27 ror x20, x13, #54 - sli v4.2d, v24.2d, #14 + sli V4.2D, V24.2D, #14 ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b + eor V31.16B, V21.16B, V26.16B ror x9, x12, #61 ; Row Mix Base - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 + ushr V24.2D, V31.2D, #62 bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 + ushr V21.2D, V8.2D, #9 bic x28, x2, x6 - sli v24.2d, v31.2d, #2 + sli V24.2D, V31.2D, #2 bic x30, x3, x2 - sli v21.2d, v8.2d, #55 + sli V21.2D, V8.2D, #55 eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b + eor V31.16B, V16.16B, V26.16B eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 + ushr V8.2D, V31.2D, #19 eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 + ushr V16.2D, V5.2D, #28 eor x4, x4, x12 - sli v8.2d, v31.2d, #45 + sli V8.2D, V31.2D, #45 eor x6, x6, x30 - sli v16.2d, v5.2d, #36 + sli V16.2D, V5.2D, #36 bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b + eor V31.16B, V3.16B, V28.16B bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b + eor V18.16B, V18.16B, V28.16B bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 + ushr V5.2D, V31.2D, #36 bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 + ushr V3.2D, V18.2D, #43 eor x7, x7, x12 - sli v5.2d, v31.2d, #28 + sli V5.2D, V31.2D, #28 eor x8, x8, x1 - sli v3.2d, v18.2d, #21 + sli V3.2D, V18.2D, #21 bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b + eor V31.16B, V17.16B, V27.16B eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b + eor V11.16B, V11.16B, V26.16B eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 + ushr V18.2D, V31.2D, #49 eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 + ushr V17.2D, V11.2D, #54 bic x12, x14, x13 - sli v18.2d, v31.2d, #15 + sli V18.2D, V31.2D, #15 bic x1, x15, x14 - sli v17.2d, v11.2d, #10 + sli V17.2D, V11.2D, #10 bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b + eor V31.16B, V7.16B, V27.16B bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b + eor V10.16B, V10.16B, V25.16B eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 + ushr V11.2D, V31.2D, #58 eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 + ushr V7.2D, V10.2D, #61 bic x0, x16, x15 - sli v11.2d, v31.2d, #6 + sli V11.2D, V31.2D, #6 eor x15, x15, x28 - sli v7.2d, v10.2d, #3 + sli V7.2D, V10.2D, #3 eor x14, x14, x0 ; Row Mix NEON - bic v25.16b, v2.16b, v1.16b + bic V25.16B, V2.16B, V1.16B eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b + bic V26.16B, V3.16B, V2.16B bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b + bic V27.16B, V4.16B, V3.16B bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b + bic V28.16B, V0.16B, V4.16B bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b + bic V29.16B, V1.16B, V0.16B bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b + eor V0.16B, V0.16B, V25.16B eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b + eor V1.16B, V1.16B, V26.16B eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b + eor V2.16B, V2.16B, V27.16B bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b + eor V3.16B, V3.16B, V28.16B eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b + eor V4.16B, V4.16B, V29.16B eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b + bic V25.16B, V7.16B, V6.16B eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b + bic V26.16B, V8.16B, V7.16B bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b + bic V27.16B, V9.16B, V8.16B bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b + bic V28.16B, V5.16B, V9.16B bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b + bic V29.16B, V6.16B, V5.16B bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b + eor V7.16B, V7.16B, V27.16B bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b + eor V9.16B, V9.16B, V29.16B eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b + bic V25.16B, V12.16B, V11.16B eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B ; Done transforming ldp x28, x1, [x29, #48] ldr x0, [x28], #8 subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_shake128_blocksx3_seed_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x2, x3, [x0] stp x4, x5, [x0, #16] @@ -10997,10 +10997,10 @@ L_SHA3_shake128_blocksx3_seed_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP @@ -11016,384 +11016,384 @@ mlkem_shake256_blocksx3_seed_neon PROC stp x24, x25, [x29, #120] stp x26, x27, [x29, #136] str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] adrp x28, L_sha3_aarch64_r add x28, x28, L_sha3_aarch64_r str x0, [x29, #40] add x0, x0, #32 - ld1 {v4.d}[0], [x0] + ld1 {V4.D}[0], [x0] ldp x2, x3, [x1], #16 add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] + ld1 {V4.D}[1], [x0] ldp x4, x5, [x1], #16 ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b + eor V5.16B, V5.16B, V5.16B eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b + eor V6.16B, V6.16B, V6.16B eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b + eor V7.16B, V7.16B, V7.16B eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b + eor V8.16B, V8.16B, V8.16B eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b + eor V9.16B, V9.16B, V9.16B eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b + eor V10.16B, V10.16B, V10.16B eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b + eor V11.16B, V11.16B, V11.16B eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b + eor V12.16B, V12.16B, V12.16B eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b + eor V13.16B, V13.16B, V13.16B eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b + eor V14.16B, V14.16B, V14.16B eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b + eor V15.16B, V15.16B, V15.16B eor x17, x17, x17 movz x19, #0x8000, lsl 48 - eor v17.16b, v17.16b, v17.16b + eor V17.16B, V17.16B, V17.16B eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b + eor V18.16B, V18.16B, V18.16B eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b + eor V19.16B, V19.16B, V19.16B eor x22, x22, x22 - eor v20.16b, v20.16b, v20.16b + eor V20.16B, V20.16B, V20.16B eor x23, x23, x23 - eor v21.16b, v21.16b, v21.16b + eor V21.16B, V21.16B, V21.16B eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b + eor V22.16B, V22.16B, V22.16B eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b + eor V23.16B, V23.16B, V23.16B eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b + eor V24.16B, V24.16B, V24.16B eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v16.2d, x19 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V16.2D, x19 mov x1, #24 ; Start of 24 rounds L_SHA3_shake256_blocksx3_seed_neon_begin stp x28, x1, [x29, #48] ; Col Mix NEON - eor v30.16b, v4.16b, v9.16b + eor V30.16B, V4.16B, V9.16B eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b + eor V27.16B, V1.16B, V6.16B eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b + eor V30.16B, V30.16B, V14.16B eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b + eor V27.16B, V27.16B, V11.16B eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b + eor V30.16B, V30.16B, V19.16B eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b + eor V27.16B, V27.16B, V16.16B eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b + eor V30.16B, V30.16B, V24.16B eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b + eor V27.16B, V27.16B, V21.16B eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 + ushr V25.2D, V27.2D, #63 eor x28, x28, x20 - sli v25.2d, v27.2d, #1 + sli V25.2D, V27.2D, #1 eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b + eor V25.16B, V25.16B, V30.16B eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b + eor V31.16B, V0.16B, V5.16B eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b + eor V28.16B, V2.16B, V7.16B str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b + eor V31.16B, V31.16B, V10.16B str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b + eor V28.16B, V28.16B, V12.16B eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b + eor V31.16B, V31.16B, V15.16B eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b + eor V28.16B, V28.16B, V17.16B eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b + eor V31.16B, V31.16B, V20.16B eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b + eor V28.16B, V28.16B, V22.16B eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 + ushr V29.2D, V30.2D, #63 eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 + ushr V26.2D, V28.2D, #63 eor x1, x1, x24 - sli v29.2d, v30.2d, #1 + sli V29.2D, V30.2D, #1 eor x28, x28, x26 - sli v26.2d, v28.2d, #1 + sli V26.2D, V28.2D, #1 eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b + eor V28.16B, V28.16B, V29.16B eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b + eor V29.16B, V3.16B, V8.16B eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b + eor V26.16B, V26.16B, V31.16B eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b + eor V29.16B, V29.16B, V13.16B eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b + eor V29.16B, V29.16B, V18.16B eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b + eor V29.16B, V29.16B, V23.16B eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 + ushr V30.2D, V29.2D, #63 eor x4, x4, x1 - sli v30.2d, v29.2d, #1 + sli V30.2D, V29.2D, #1 eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b + eor V27.16B, V27.16B, V30.16B eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x20, x20, x1 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b + eor V29.16B, V29.16B, V30.16B ldr x0, [x29, #32] ; Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 + ushr V30.2D, V31.2D, #63 eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 + ushr V1.2D, V6.2D, #20 eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 + sli V30.2D, V31.2D, #1 eor x6, x6, x28 - sli v1.2d, v6.2d, #44 + sli V1.2D, V6.2D, #44 eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b + eor V31.16B, V9.16B, V29.16B eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b + eor V22.16B, V22.16B, V27.16B eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 + ushr V6.2D, V31.2D, #44 eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 + ushr V9.2D, V22.2D, #3 eor x3, x3, x30 - sli v6.2d, v31.2d, #20 + sli V6.2D, V31.2D, #20 eor x8, x8, x30 - sli v9.2d, v22.2d, #61 + sli V9.2D, V22.2D, #61 eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b + eor V31.16B, V14.16B, V29.16B eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b + eor V20.16B, V20.16B, V25.16B eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 + ushr V22.2D, V31.2D, #25 eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 + ushr V14.2D, V20.2D, #46 eor x10, x10, x1 - sli v22.2d, v31.2d, #39 + sli V22.2D, V31.2D, #39 eor x15, x15, x1 - sli v14.2d, v20.2d, #18 + sli V14.2D, V20.2D, #18 eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b + eor V31.16B, V2.16B, V27.16B eor x26, x26, x1 ; Swap Rotate Base - eor v12.16b, v12.16b, v27.16b + eor V12.16B, V12.16B, V27.16B ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 + ushr V20.2D, V31.2D, #2 ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 + ushr V2.2D, V12.2D, #21 ror x8, x11, #44 - sli v20.2d, v31.2d, #62 + sli V20.2D, V31.2D, #62 ror x11, x25, #3 - sli v2.2d, v12.2d, #43 + sli V2.2D, V12.2D, #43 ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b + eor V31.16B, V13.16B, V28.16B ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b + eor V19.16B, V19.16B, V29.16B ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 + ushr V12.2D, V31.2D, #39 ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 + ushr V13.2D, V19.2D, #56 ror x14, x15, #39 - sli v12.2d, v31.2d, #25 + sli V12.2D, V31.2D, #25 ror x15, x22, #56 - sli v13.2d, v19.2d, #8 + sli V13.2D, V19.2D, #8 ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b + eor V31.16B, V23.16B, V28.16B ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b + eor V15.16B, V15.16B, V25.16B ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 + ushr V19.2D, V31.2D, #8 ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 + ushr V23.2D, V15.2D, #23 ror x27, x24, #62 - sli v19.2d, v31.2d, #56 + sli V19.2D, V31.2D, #56 ror x24, x10, #9 - sli v23.2d, v15.2d, #41 + sli V23.2D, V15.2D, #41 ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b + eor V31.16B, V4.16B, V29.16B ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b + eor V24.16B, V24.16B, V29.16B ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 + ushr V15.2D, V31.2D, #37 ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 + ushr V4.2D, V24.2D, #50 ror x21, x20, #49 - sli v15.2d, v31.2d, #27 + sli V15.2D, V31.2D, #27 ror x20, x13, #54 - sli v4.2d, v24.2d, #14 + sli V4.2D, V24.2D, #14 ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b + eor V31.16B, V21.16B, V26.16B ror x9, x12, #61 ; Row Mix Base - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 + ushr V24.2D, V31.2D, #62 bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 + ushr V21.2D, V8.2D, #9 bic x28, x2, x6 - sli v24.2d, v31.2d, #2 + sli V24.2D, V31.2D, #2 bic x30, x3, x2 - sli v21.2d, v8.2d, #55 + sli V21.2D, V8.2D, #55 eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b + eor V31.16B, V16.16B, V26.16B eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 + ushr V8.2D, V31.2D, #19 eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 + ushr V16.2D, V5.2D, #28 eor x4, x4, x12 - sli v8.2d, v31.2d, #45 + sli V8.2D, V31.2D, #45 eor x6, x6, x30 - sli v16.2d, v5.2d, #36 + sli V16.2D, V5.2D, #36 bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b + eor V31.16B, V3.16B, V28.16B bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b + eor V18.16B, V18.16B, V28.16B bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 + ushr V5.2D, V31.2D, #36 bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 + ushr V3.2D, V18.2D, #43 eor x7, x7, x12 - sli v5.2d, v31.2d, #28 + sli V5.2D, V31.2D, #28 eor x8, x8, x1 - sli v3.2d, v18.2d, #21 + sli V3.2D, V18.2D, #21 bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b + eor V31.16B, V17.16B, V27.16B eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b + eor V11.16B, V11.16B, V26.16B eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 + ushr V18.2D, V31.2D, #49 eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 + ushr V17.2D, V11.2D, #54 bic x12, x14, x13 - sli v18.2d, v31.2d, #15 + sli V18.2D, V31.2D, #15 bic x1, x15, x14 - sli v17.2d, v11.2d, #10 + sli V17.2D, V11.2D, #10 bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b + eor V31.16B, V7.16B, V27.16B bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b + eor V10.16B, V10.16B, V25.16B eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 + ushr V11.2D, V31.2D, #58 eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 + ushr V7.2D, V10.2D, #61 bic x0, x16, x15 - sli v11.2d, v31.2d, #6 + sli V11.2D, V31.2D, #6 eor x15, x15, x28 - sli v7.2d, v10.2d, #3 + sli V7.2D, V10.2D, #3 eor x14, x14, x0 ; Row Mix NEON - bic v25.16b, v2.16b, v1.16b + bic V25.16B, V2.16B, V1.16B eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b + bic V26.16B, V3.16B, V2.16B bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b + bic V27.16B, V4.16B, V3.16B bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b + bic V28.16B, V0.16B, V4.16B bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b + bic V29.16B, V1.16B, V0.16B bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b + eor V0.16B, V0.16B, V25.16B eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b + eor V1.16B, V1.16B, V26.16B eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b + eor V2.16B, V2.16B, V27.16B bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b + eor V3.16B, V3.16B, V28.16B eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b + eor V4.16B, V4.16B, V29.16B eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b + bic V25.16B, V7.16B, V6.16B eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b + bic V26.16B, V8.16B, V7.16B bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b + bic V27.16B, V9.16B, V8.16B bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b + bic V28.16B, V5.16B, V9.16B bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b + bic V29.16B, V6.16B, V5.16B bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b + eor V5.16B, V5.16B, V25.16B eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b + eor V6.16B, V6.16B, V26.16B eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b + eor V7.16B, V7.16B, V27.16B bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b + eor V8.16B, V8.16B, V28.16B eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b + eor V9.16B, V9.16B, V29.16B eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b + bic V25.16B, V12.16B, V11.16B eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B ; Done transforming ldp x28, x1, [x29, #48] ldr x0, [x28], #8 subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 + mov V30.D[0], x0 + mov V30.D[1], x0 eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_SHA3_shake256_blocksx3_seed_neon_begin ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] add x0, x0, #8 stp x2, x3, [x0] stp x4, x5, [x0, #16] @@ -11414,10 +11414,10 @@ L_SHA3_shake256_blocksx3_seed_neon_begin ldp x24, x25, [x29, #120] ldp x26, x27, [x29, #136] ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] ldp x29, x30, [sp], #0xe0 ret ENDP diff --git a/wolfcrypt/src/port/arm/armv8-poly1305-asm.asm b/wolfcrypt/src/port/arm/armv8-poly1305-asm.asm index 61940944b3..4ee10c516e 100644 --- a/wolfcrypt/src/port/arm/armv8-poly1305-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-poly1305-asm.asm @@ -106,221 +106,221 @@ poly1305_arm64_blocks PROC stp x29, x30, [sp, #-96]! add x29, sp, #0 str x17, [x29, #24] - stp d8, d9, [x29, #32] - stp d10, d11, [x29, #48] - stp d12, d13, [x29, #64] - stp d14, d15, [x29, #80] + stp D8, D9, [x29, #32] + stp D10, D11, [x29, #48] + stp D12, D13, [x29, #64] + stp D14, D15, [x29, #80] cmp x2, #0x40 blt L_poly1305_arm64_blocks_done ; Set mask (0x3ffffff), hi bit and 5 into vector registers - movi v25.16b, #0xff - movi v27.4s, #1, lsl 24 - ushr v25.4s, v25.4s, #6 - movi v24.4s, #5 - uxtl v26.2d, v25.2s + movi V25.16B, #0xff + movi V27.4S, #1, lsl 24 + ushr V25.4S, V25.4S, #6 + movi V24.4S, #5 + ushll V26.2D, V25.2S, #0 add x14, x0, #16 - ld4 {v15.4s, v16.4s, v17.4s, v18.4s}, [x14], #0x40 - ld1 {v19.4s}, [x14] + ld4 {V15.4S, V16.4S, V17.4S, V18.4S}, [x14], #0x40 + ld1 {V19.4S}, [x14] add x14, x0, #0x60 - movi v0.4s, #0 - movi v1.4s, #0 - movi v2.4s, #0 - movi v3.4s, #0 - movi v4.4s, #0 - ld4 {v0.s, v1.s, v2.s, v3.s}[0], [x14], #16 - ld1 {v4.s}[0], [x14] - mul v20.4s, v16.4s, v24.4s - mul v21.4s, v17.4s, v24.4s - mul v22.4s, v18.4s, v24.4s - mul v23.4s, v19.4s, v24.4s + movi V0.4S, #0 + movi V1.4S, #0 + movi V2.4S, #0 + movi V3.4S, #0 + movi V4.4S, #0 + ld4 {V0.S, V1.S, V2.S, V3.S}[0], [x14], #16 + ld1 {V4.S}[0], [x14] + mul V20.4S, V16.4S, V24.4S + mul V21.4S, V17.4S, V24.4S + mul V22.4S, V18.4S, V24.4S + mul V23.4S, V19.4S, V24.4S L_poly1305_arm64_blocks_loop_64 ; Load message of 64 bytes - setting hi bit for not finished - ld4 {v5.4s, v6.4s, v7.4s, v8.4s}, [x1], #0x40 + ld4 {V5.4S, V6.4S, V7.4S, V8.4S}, [x1], #0x40 sub x2, x2, #0x40 - ushr v9.4s, v8.4s, #8 - shl v8.4s, v8.4s, #18 - orr v9.16b, v9.16b, v27.16b - sri v8.4s, v7.4s, #14 - shl v7.4s, v7.4s, #12 - and v8.16b, v8.16b, v25.16b - sri v7.4s, v6.4s, #20 - shl v6.4s, v6.4s, #6 - and v7.16b, v7.16b, v25.16b - sri v6.4s, v5.4s, #26 - and v5.16b, v5.16b, v25.16b - and v6.16b, v6.16b, v25.16b - umull2 v10.2d, v5.4s, v15.4s - umull2 v11.2d, v5.4s, v16.4s - umull2 v12.2d, v5.4s, v17.4s - umull2 v13.2d, v5.4s, v18.4s - umull2 v14.2d, v5.4s, v19.4s - umlal2 v10.2d, v6.4s, v23.4s - umlal2 v11.2d, v6.4s, v15.4s - umlal2 v12.2d, v6.4s, v16.4s - umlal2 v13.2d, v6.4s, v17.4s - umlal2 v14.2d, v6.4s, v18.4s - umlal2 v10.2d, v7.4s, v22.4s - umlal2 v11.2d, v7.4s, v23.4s - umlal2 v12.2d, v7.4s, v15.4s - umlal2 v13.2d, v7.4s, v16.4s - umlal2 v14.2d, v7.4s, v17.4s - umlal2 v10.2d, v8.4s, v21.4s - umlal2 v11.2d, v8.4s, v22.4s - umlal2 v12.2d, v8.4s, v23.4s - umlal2 v13.2d, v8.4s, v15.4s - umlal2 v14.2d, v8.4s, v16.4s - umlal2 v10.2d, v9.4s, v20.4s - umlal2 v11.2d, v9.4s, v21.4s - umlal2 v12.2d, v9.4s, v22.4s - umlal2 v13.2d, v9.4s, v23.4s - umlal2 v14.2d, v9.4s, v15.4s - add v5.4s, v5.4s, v0.4s - add v6.4s, v6.4s, v1.4s - add v7.4s, v7.4s, v2.4s - add v8.4s, v8.4s, v3.4s - add v9.4s, v9.4s, v4.4s - umlal v10.2d, v5.2s, v15.2s - umlal v11.2d, v5.2s, v16.2s - umlal v12.2d, v5.2s, v17.2s - umlal v13.2d, v5.2s, v18.2s - umlal v14.2d, v5.2s, v19.2s - umlal v10.2d, v6.2s, v23.2s - umlal v11.2d, v6.2s, v15.2s - umlal v12.2d, v6.2s, v16.2s - umlal v13.2d, v6.2s, v17.2s - umlal v14.2d, v6.2s, v18.2s - umlal v10.2d, v7.2s, v22.2s - umlal v11.2d, v7.2s, v23.2s - umlal v12.2d, v7.2s, v15.2s - umlal v13.2d, v7.2s, v16.2s - umlal v14.2d, v7.2s, v17.2s - umlal v10.2d, v8.2s, v21.2s - umlal v11.2d, v8.2s, v22.2s - umlal v12.2d, v8.2s, v23.2s - umlal v13.2d, v8.2s, v15.2s - umlal v14.2d, v8.2s, v16.2s - umlal v10.2d, v9.2s, v20.2s - umlal v11.2d, v9.2s, v21.2s - umlal v12.2d, v9.2s, v22.2s - umlal v13.2d, v9.2s, v23.2s - umlal v14.2d, v9.2s, v15.2s - addp d10, v10.2d - addp d11, v11.2d - addp d12, v12.2d - addp d13, v13.2d - addp d14, v14.2d + ushr V9.4S, V8.4S, #8 + shl V8.4S, V8.4S, #18 + orr V9.16B, V9.16B, V27.16B + sri V8.4S, V7.4S, #14 + shl V7.4S, V7.4S, #12 + and V8.16B, V8.16B, V25.16B + sri V7.4S, V6.4S, #20 + shl V6.4S, V6.4S, #6 + and V7.16B, V7.16B, V25.16B + sri V6.4S, V5.4S, #26 + and V5.16B, V5.16B, V25.16B + and V6.16B, V6.16B, V25.16B + umull2 V10.2D, V5.4S, V15.4S + umull2 V11.2D, V5.4S, V16.4S + umull2 V12.2D, V5.4S, V17.4S + umull2 V13.2D, V5.4S, V18.4S + umull2 V14.2D, V5.4S, V19.4S + umlal2 V10.2D, V6.4S, V23.4S + umlal2 V11.2D, V6.4S, V15.4S + umlal2 V12.2D, V6.4S, V16.4S + umlal2 V13.2D, V6.4S, V17.4S + umlal2 V14.2D, V6.4S, V18.4S + umlal2 V10.2D, V7.4S, V22.4S + umlal2 V11.2D, V7.4S, V23.4S + umlal2 V12.2D, V7.4S, V15.4S + umlal2 V13.2D, V7.4S, V16.4S + umlal2 V14.2D, V7.4S, V17.4S + umlal2 V10.2D, V8.4S, V21.4S + umlal2 V11.2D, V8.4S, V22.4S + umlal2 V12.2D, V8.4S, V23.4S + umlal2 V13.2D, V8.4S, V15.4S + umlal2 V14.2D, V8.4S, V16.4S + umlal2 V10.2D, V9.4S, V20.4S + umlal2 V11.2D, V9.4S, V21.4S + umlal2 V12.2D, V9.4S, V22.4S + umlal2 V13.2D, V9.4S, V23.4S + umlal2 V14.2D, V9.4S, V15.4S + add V5.4S, V5.4S, V0.4S + add V6.4S, V6.4S, V1.4S + add V7.4S, V7.4S, V2.4S + add V8.4S, V8.4S, V3.4S + add V9.4S, V9.4S, V4.4S + umlal V10.2D, V5.2S, V15.2S + umlal V11.2D, V5.2S, V16.2S + umlal V12.2D, V5.2S, V17.2S + umlal V13.2D, V5.2S, V18.2S + umlal V14.2D, V5.2S, V19.2S + umlal V10.2D, V6.2S, V23.2S + umlal V11.2D, V6.2S, V15.2S + umlal V12.2D, V6.2S, V16.2S + umlal V13.2D, V6.2S, V17.2S + umlal V14.2D, V6.2S, V18.2S + umlal V10.2D, V7.2S, V22.2S + umlal V11.2D, V7.2S, V23.2S + umlal V12.2D, V7.2S, V15.2S + umlal V13.2D, V7.2S, V16.2S + umlal V14.2D, V7.2S, V17.2S + umlal V10.2D, V8.2S, V21.2S + umlal V11.2D, V8.2S, V22.2S + umlal V12.2D, V8.2S, V23.2S + umlal V13.2D, V8.2S, V15.2S + umlal V14.2D, V8.2S, V16.2S + umlal V10.2D, V9.2S, V20.2S + umlal V11.2D, V9.2S, V21.2S + umlal V12.2D, V9.2S, V22.2S + umlal V13.2D, V9.2S, V23.2S + umlal V14.2D, V9.2S, V15.2S + addp D10, V10.2D + addp D11, V11.2D + addp D12, V12.2D + addp D13, V13.2D + addp D14, V14.2D ; Redistribute and handle overflow - usra v11.2d, v10.2d, #26 - and v10.16b, v10.16b, v26.16b - usra v14.2d, v13.2d, #26 - and v3.16b, v13.16b, v26.16b - ushr v2.2d, v14.2d, #26 - usra v12.2d, v11.2d, #26 - shl v0.2d, v2.2d, #2 - and v1.16b, v11.16b, v26.16b - add v0.2d, v0.2d, v2.2d - and v4.16b, v14.16b, v26.16b - add v10.2d, v10.2d, v0.2d - usra v3.2d, v12.2d, #26 - and v2.16b, v12.16b, v26.16b - usra v1.2d, v10.2d, #26 - and v0.16b, v10.16b, v26.16b - usra v4.2d, v3.2d, #26 - and v3.16b, v3.16b, v26.16b + usra V11.2D, V10.2D, #26 + and V10.16B, V10.16B, V26.16B + usra V14.2D, V13.2D, #26 + and V3.16B, V13.16B, V26.16B + ushr V2.2D, V14.2D, #26 + usra V12.2D, V11.2D, #26 + shl V0.2D, V2.2D, #2 + and V1.16B, V11.16B, V26.16B + add V0.2D, V0.2D, V2.2D + and V4.16B, V14.16B, V26.16B + add V10.2D, V10.2D, V0.2D + usra V3.2D, V12.2D, #26 + and V2.16B, V12.16B, V26.16B + usra V1.2D, V10.2D, #26 + and V0.16B, V10.16B, V26.16B + usra V4.2D, V3.2D, #26 + and V3.16B, V3.16B, V26.16B cmp x2, #0x40 bge L_poly1305_arm64_blocks_loop_64 cmp x2, #16 ble L_poly1305_arm64_blocks_done_32 ; Start 32 - ld4 {v5.2s, v6.2s, v7.2s, v8.2s}, [x1], #32 + ld4 {V5.2S, V6.2S, V7.2S, V8.2S}, [x1], #32 sub x2, x2, #32 - mov v15.d[0], v15.d[1] - mov v16.d[0], v16.d[1] - mov v17.d[0], v17.d[1] - mov v18.d[0], v18.d[1] - mov v19.d[0], v19.d[1] - mov v20.d[0], v20.d[1] - mov v21.d[0], v21.d[1] - mov v22.d[0], v22.d[1] - mov v23.d[0], v23.d[1] - ushr v9.2s, v8.2s, #8 - shl v8.2s, v8.2s, #18 - orr v9.8b, v9.8b, v27.8b - sri v8.2s, v7.2s, #14 - shl v7.2s, v7.2s, #12 - and v8.8b, v8.8b, v25.8b - sri v7.2s, v6.2s, #20 - shl v6.2s, v6.2s, #6 - and v7.8b, v7.8b, v25.8b - sri v6.2s, v5.2s, #26 - and v5.8b, v5.8b, v25.8b - and v6.8b, v6.8b, v25.8b - add v5.2s, v5.2s, v0.2s - add v6.2s, v6.2s, v1.2s - add v7.2s, v7.2s, v2.2s - add v8.2s, v8.2s, v3.2s - add v9.2s, v9.2s, v4.2s - umull v10.2d, v5.2s, v15.2s - umull v11.2d, v5.2s, v16.2s - umull v12.2d, v5.2s, v17.2s - umull v13.2d, v5.2s, v18.2s - umull v14.2d, v5.2s, v19.2s - umlal v10.2d, v6.2s, v23.2s - umlal v11.2d, v6.2s, v15.2s - umlal v12.2d, v6.2s, v16.2s - umlal v13.2d, v6.2s, v17.2s - umlal v14.2d, v6.2s, v18.2s - umlal v10.2d, v7.2s, v22.2s - umlal v11.2d, v7.2s, v23.2s - umlal v12.2d, v7.2s, v15.2s - umlal v13.2d, v7.2s, v16.2s - umlal v14.2d, v7.2s, v17.2s - umlal v10.2d, v8.2s, v21.2s - umlal v11.2d, v8.2s, v22.2s - umlal v12.2d, v8.2s, v23.2s - umlal v13.2d, v8.2s, v15.2s - umlal v14.2d, v8.2s, v16.2s - umlal v10.2d, v9.2s, v20.2s - umlal v11.2d, v9.2s, v21.2s - umlal v12.2d, v9.2s, v22.2s - umlal v13.2d, v9.2s, v23.2s - umlal v14.2d, v9.2s, v15.2s - addp d10, v10.2d - addp d11, v11.2d - addp d12, v12.2d - addp d13, v13.2d - addp d14, v14.2d + mov V15.D[0], V15.D[1] + mov V16.D[0], V16.D[1] + mov V17.D[0], V17.D[1] + mov V18.D[0], V18.D[1] + mov V19.D[0], V19.D[1] + mov V20.D[0], V20.D[1] + mov V21.D[0], V21.D[1] + mov V22.D[0], V22.D[1] + mov V23.D[0], V23.D[1] + ushr V9.2S, V8.2S, #8 + shl V8.2S, V8.2S, #18 + orr V9.8B, V9.8B, V27.8B + sri V8.2S, V7.2S, #14 + shl V7.2S, V7.2S, #12 + and V8.8B, V8.8B, V25.8B + sri V7.2S, V6.2S, #20 + shl V6.2S, V6.2S, #6 + and V7.8B, V7.8B, V25.8B + sri V6.2S, V5.2S, #26 + and V5.8B, V5.8B, V25.8B + and V6.8B, V6.8B, V25.8B + add V5.2S, V5.2S, V0.2S + add V6.2S, V6.2S, V1.2S + add V7.2S, V7.2S, V2.2S + add V8.2S, V8.2S, V3.2S + add V9.2S, V9.2S, V4.2S + umull V10.2D, V5.2S, V15.2S + umull V11.2D, V5.2S, V16.2S + umull V12.2D, V5.2S, V17.2S + umull V13.2D, V5.2S, V18.2S + umull V14.2D, V5.2S, V19.2S + umlal V10.2D, V6.2S, V23.2S + umlal V11.2D, V6.2S, V15.2S + umlal V12.2D, V6.2S, V16.2S + umlal V13.2D, V6.2S, V17.2S + umlal V14.2D, V6.2S, V18.2S + umlal V10.2D, V7.2S, V22.2S + umlal V11.2D, V7.2S, V23.2S + umlal V12.2D, V7.2S, V15.2S + umlal V13.2D, V7.2S, V16.2S + umlal V14.2D, V7.2S, V17.2S + umlal V10.2D, V8.2S, V21.2S + umlal V11.2D, V8.2S, V22.2S + umlal V12.2D, V8.2S, V23.2S + umlal V13.2D, V8.2S, V15.2S + umlal V14.2D, V8.2S, V16.2S + umlal V10.2D, V9.2S, V20.2S + umlal V11.2D, V9.2S, V21.2S + umlal V12.2D, V9.2S, V22.2S + umlal V13.2D, V9.2S, V23.2S + umlal V14.2D, V9.2S, V15.2S + addp D10, V10.2D + addp D11, V11.2D + addp D12, V12.2D + addp D13, V13.2D + addp D14, V14.2D ; Redistribute and handle overflow - usra v11.2d, v10.2d, #26 - and v10.16b, v10.16b, v26.16b - usra v14.2d, v13.2d, #26 - and v3.16b, v13.16b, v26.16b - ushr v2.2d, v14.2d, #26 - usra v12.2d, v11.2d, #26 - shl v0.2d, v2.2d, #2 - and v1.16b, v11.16b, v26.16b - add v0.2d, v0.2d, v2.2d - and v4.16b, v14.16b, v26.16b - add v10.2d, v10.2d, v0.2d - usra v3.2d, v12.2d, #26 - and v2.16b, v12.16b, v26.16b - usra v1.2d, v10.2d, #26 - and v0.16b, v10.16b, v26.16b - usra v4.2d, v3.2d, #26 - and v3.16b, v3.16b, v26.16b + usra V11.2D, V10.2D, #26 + and V10.16B, V10.16B, V26.16B + usra V14.2D, V13.2D, #26 + and V3.16B, V13.16B, V26.16B + ushr V2.2D, V14.2D, #26 + usra V12.2D, V11.2D, #26 + shl V0.2D, V2.2D, #2 + and V1.16B, V11.16B, V26.16B + add V0.2D, V0.2D, V2.2D + and V4.16B, V14.16B, V26.16B + add V10.2D, V10.2D, V0.2D + usra V3.2D, V12.2D, #26 + and V2.16B, V12.16B, V26.16B + usra V1.2D, V10.2D, #26 + and V0.16B, V10.16B, V26.16B + usra V4.2D, V3.2D, #26 + and V3.16B, V3.16B, V26.16B L_poly1305_arm64_blocks_done_32 cmp x2, #16 beq L_poly1305_arm64_blocks_transfer add x14, x0, #0x60 - st4 {v0.s, v1.s, v2.s, v3.s}[0], [x14], #16 - st1 {v4.s}[0], [x14] + st4 {V0.S, V1.S, V2.S, V3.S}[0], [x14], #16 + st1 {V4.S}[0], [x14] b L_poly1305_arm64_blocks_done_all L_poly1305_arm64_blocks_transfer - mov w3, v0.s[0] - mov w4, v1.s[0] - mov w5, v2.s[0] - mov w6, v3.s[0] - mov w7, v4.s[0] + mov w3, V0.S[0] + mov w4, V1.S[0] + mov w5, V2.S[0] + mov w6, V3.S[0] + mov w7, V4.S[0] b L_poly1305_arm64_blocks_start L_poly1305_arm64_blocks_done cmp x2, #16 @@ -404,14 +404,14 @@ L_poly1305_arm64_blocks_loop str w7, [x0, #112] L_poly1305_arm64_blocks_done_all ldr x17, [x29, #24] - ldp d8, d9, [x29, #32] - ldp d10, d11, [x29, #48] - ldp d12, d13, [x29, #64] - ldp d14, d15, [x29, #80] + ldp D8, D9, [x29, #32] + ldp D10, D11, [x29, #48] + ldp D12, D13, [x29, #64] + ldp D14, D15, [x29, #80] ldp x29, x30, [sp], #0x60 ret ENDP - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_poly1305_set_key_arm64_clamp DCD 0x0fffffff, 0x0ffffffc, 0x0ffffffc, 0x0ffffffc diff --git a/wolfcrypt/src/port/arm/armv8-sha256-asm.asm b/wolfcrypt/src/port/arm/armv8-sha256-asm.asm index 877ff66414..42bdd86682 100644 --- a/wolfcrypt/src/port/arm/armv8-sha256-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-sha256-asm.asm @@ -25,7 +25,7 @@ ; ruby ./sha2/sha256.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-sha256-asm.asm IF :LNOT::DEF:NO_SHA256 :LOR: :DEF:WOLFSSL_SHA224 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_SHA256_transform_neon_len_k DCD 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 @@ -54,8 +54,8 @@ Transform_Sha256_Len_neon PROC stp x20, x21, [x29, #40] stp x22, x23, [x29, #56] str x24, [x29, #72] - stp d8, d9, [x29, #80] - stp d10, d11, [x29, #96] + stp D8, D9, [x29, #80] + stp D10, D11, [x29, #96] adrp x3, L_SHA256_transform_neon_len_k add x3, x3, L_SHA256_transform_neon_len_k ; Load digest into working vars @@ -71,29 +71,29 @@ Transform_Sha256_Len_neon PROC L_sha256_len_neon_begin ; Load W ; Copy digest to add in at end - ld1 {v0.8b, v1.8b, v2.8b, v3.8b}, [x1], #32 + ld1 {V0.8B, V1.8B, V2.8B, V3.8B}, [x1], #32 mov w15, w4 - ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [x1], #32 + ld1 {V4.8B, V5.8B, V6.8B, V7.8B}, [x1], #32 mov w16, w5 - rev32 v0.8b, v0.8b + rev32 V0.8B, V0.8B mov w17, w6 - rev32 v1.8b, v1.8b + rev32 V1.8B, V1.8B mov w19, w7 - rev32 v2.8b, v2.8b + rev32 V2.8B, V2.8B mov w20, w8 - rev32 v3.8b, v3.8b + rev32 V3.8B, V3.8B mov w21, w9 - rev32 v4.8b, v4.8b + rev32 V4.8B, V4.8B mov w22, w10 - rev32 v5.8b, v5.8b + rev32 V5.8B, V5.8B mov w23, w11 - rev32 v6.8b, v6.8b - rev32 v7.8b, v7.8b + rev32 V6.8B, V6.8B + rev32 V7.8B, V7.8B mov x24, #3 ; Start of 16 rounds L_sha256_len_neon_start ; Round 0 - mov w14, v0.s[0] + mov w14, V0.S[0] ror w12, w8, #6 eor w13, w9, w10 eor w12, w12, w8, ror 11 @@ -116,50 +116,50 @@ L_sha256_len_neon_start add w11, w11, w12 add w11, w11, w13 ; Round 1 - mov w14, v0.s[1] + mov w14, V0.S[1] ; Calc new W[0]-W[1] - ext v10.8b, v0.8b, v1.8b, #4 + ext8 V10.8B, V0.8B, V1.8B, #4 ror w12, w7, #6 - shl v8.2s, v7.2s, #15 + shl V8.2S, V7.2S, #15 eor w13, w8, w9 - sri v8.2s, v7.2s, #17 + sri V8.2S, V7.2S, #17 eor w12, w12, w7, ror 11 - shl v9.2s, v7.2s, #13 + shl V9.2S, V7.2S, #13 and w13, w13, w7 - sri v9.2s, v7.2s, #19 + sri V9.2S, V7.2S, #19 eor w12, w12, w7, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w9 - ushr v8.2s, v7.2s, #10 + ushr V8.2S, V7.2S, #10 add w10, w10, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w10, w10, w13 - add v0.2s, v0.2s, v9.2s + add V0.2S, V0.2S, V9.2S ldr w12, [x3, #4] - ext v11.8b, v4.8b, v5.8b, #4 + ext8 V11.8B, V4.8B, V5.8B, #4 add w10, w10, w14 - add v0.2s, v0.2s, v11.2s + add V0.2S, V0.2S, V11.2S add w10, w10, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w6, w6, w10 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w11, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w11, w4 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w11, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w4, w5 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w11, ror 22 - add v0.2s, v0.2s, v9.2s + add V0.2S, V0.2S, V9.2S eor w13, w13, w4 add w10, w10, w12 add w10, w10, w13 ; Round 2 - mov w14, v1.s[0] + mov w14, V1.S[0] ror w12, w6, #6 eor w13, w7, w8 eor w12, w12, w6, ror 11 @@ -182,50 +182,50 @@ L_sha256_len_neon_start add w9, w9, w12 add w9, w9, w13 ; Round 3 - mov w14, v1.s[1] + mov w14, V1.S[1] ; Calc new W[2]-W[3] - ext v10.8b, v1.8b, v2.8b, #4 + ext8 V10.8B, V1.8B, V2.8B, #4 ror w12, w5, #6 - shl v8.2s, v0.2s, #15 + shl V8.2S, V0.2S, #15 eor w13, w6, w7 - sri v8.2s, v0.2s, #17 + sri V8.2S, V0.2S, #17 eor w12, w12, w5, ror 11 - shl v9.2s, v0.2s, #13 + shl V9.2S, V0.2S, #13 and w13, w13, w5 - sri v9.2s, v0.2s, #19 + sri V9.2S, V0.2S, #19 eor w12, w12, w5, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w7 - ushr v8.2s, v0.2s, #10 + ushr V8.2S, V0.2S, #10 add w8, w8, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w8, w8, w13 - add v1.2s, v1.2s, v9.2s + add V1.2S, V1.2S, V9.2S ldr w12, [x3, #12] - ext v11.8b, v5.8b, v6.8b, #4 + ext8 V11.8B, V5.8B, V6.8B, #4 add w8, w8, w14 - add v1.2s, v1.2s, v11.2s + add V1.2S, V1.2S, V11.2S add w8, w8, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w4, w4, w8 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w9, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w9, w10 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w9, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w10, w11 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w9, ror 22 - add v1.2s, v1.2s, v9.2s + add V1.2S, V1.2S, V9.2S eor w13, w13, w10 add w8, w8, w12 add w8, w8, w13 ; Round 4 - mov w14, v2.s[0] + mov w14, V2.S[0] ror w12, w4, #6 eor w13, w5, w6 eor w12, w12, w4, ror 11 @@ -248,50 +248,50 @@ L_sha256_len_neon_start add w7, w7, w12 add w7, w7, w13 ; Round 5 - mov w14, v2.s[1] + mov w14, V2.S[1] ; Calc new W[4]-W[5] - ext v10.8b, v2.8b, v3.8b, #4 + ext8 V10.8B, V2.8B, V3.8B, #4 ror w12, w11, #6 - shl v8.2s, v1.2s, #15 + shl V8.2S, V1.2S, #15 eor w13, w4, w5 - sri v8.2s, v1.2s, #17 + sri V8.2S, V1.2S, #17 eor w12, w12, w11, ror 11 - shl v9.2s, v1.2s, #13 + shl V9.2S, V1.2S, #13 and w13, w13, w11 - sri v9.2s, v1.2s, #19 + sri V9.2S, V1.2S, #19 eor w12, w12, w11, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w5 - ushr v8.2s, v1.2s, #10 + ushr V8.2S, V1.2S, #10 add w6, w6, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w6, w6, w13 - add v2.2s, v2.2s, v9.2s + add V2.2S, V2.2S, V9.2S ldr w12, [x3, #20] - ext v11.8b, v6.8b, v7.8b, #4 + ext8 V11.8B, V6.8B, V7.8B, #4 add w6, w6, w14 - add v2.2s, v2.2s, v11.2s + add V2.2S, V2.2S, V11.2S add w6, w6, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w10, w10, w6 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w7, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w7, w8 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w7, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w8, w9 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w7, ror 22 - add v2.2s, v2.2s, v9.2s + add V2.2S, V2.2S, V9.2S eor w13, w13, w8 add w6, w6, w12 add w6, w6, w13 ; Round 6 - mov w14, v3.s[0] + mov w14, V3.S[0] ror w12, w10, #6 eor w13, w11, w4 eor w12, w12, w10, ror 11 @@ -314,50 +314,50 @@ L_sha256_len_neon_start add w5, w5, w12 add w5, w5, w13 ; Round 7 - mov w14, v3.s[1] + mov w14, V3.S[1] ; Calc new W[6]-W[7] - ext v10.8b, v3.8b, v4.8b, #4 + ext8 V10.8B, V3.8B, V4.8B, #4 ror w12, w9, #6 - shl v8.2s, v2.2s, #15 + shl V8.2S, V2.2S, #15 eor w13, w10, w11 - sri v8.2s, v2.2s, #17 + sri V8.2S, V2.2S, #17 eor w12, w12, w9, ror 11 - shl v9.2s, v2.2s, #13 + shl V9.2S, V2.2S, #13 and w13, w13, w9 - sri v9.2s, v2.2s, #19 + sri V9.2S, V2.2S, #19 eor w12, w12, w9, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w11 - ushr v8.2s, v2.2s, #10 + ushr V8.2S, V2.2S, #10 add w4, w4, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w4, w4, w13 - add v3.2s, v3.2s, v9.2s + add V3.2S, V3.2S, V9.2S ldr w12, [x3, #28] - ext v11.8b, v7.8b, v0.8b, #4 + ext8 V11.8B, V7.8B, V0.8B, #4 add w4, w4, w14 - add v3.2s, v3.2s, v11.2s + add V3.2S, V3.2S, V11.2S add w4, w4, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w8, w8, w4 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w5, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w5, w6 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w5, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w6, w7 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w5, ror 22 - add v3.2s, v3.2s, v9.2s + add V3.2S, V3.2S, V9.2S eor w13, w13, w6 add w4, w4, w12 add w4, w4, w13 ; Round 8 - mov w14, v4.s[0] + mov w14, V4.S[0] ror w12, w8, #6 eor w13, w9, w10 eor w12, w12, w8, ror 11 @@ -380,50 +380,50 @@ L_sha256_len_neon_start add w11, w11, w12 add w11, w11, w13 ; Round 9 - mov w14, v4.s[1] + mov w14, V4.S[1] ; Calc new W[8]-W[9] - ext v10.8b, v4.8b, v5.8b, #4 + ext8 V10.8B, V4.8B, V5.8B, #4 ror w12, w7, #6 - shl v8.2s, v3.2s, #15 + shl V8.2S, V3.2S, #15 eor w13, w8, w9 - sri v8.2s, v3.2s, #17 + sri V8.2S, V3.2S, #17 eor w12, w12, w7, ror 11 - shl v9.2s, v3.2s, #13 + shl V9.2S, V3.2S, #13 and w13, w13, w7 - sri v9.2s, v3.2s, #19 + sri V9.2S, V3.2S, #19 eor w12, w12, w7, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w9 - ushr v8.2s, v3.2s, #10 + ushr V8.2S, V3.2S, #10 add w10, w10, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w10, w10, w13 - add v4.2s, v4.2s, v9.2s + add V4.2S, V4.2S, V9.2S ldr w12, [x3, #36] - ext v11.8b, v0.8b, v1.8b, #4 + ext8 V11.8B, V0.8B, V1.8B, #4 add w10, w10, w14 - add v4.2s, v4.2s, v11.2s + add V4.2S, V4.2S, V11.2S add w10, w10, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w6, w6, w10 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w11, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w11, w4 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w11, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w4, w5 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w11, ror 22 - add v4.2s, v4.2s, v9.2s + add V4.2S, V4.2S, V9.2S eor w13, w13, w4 add w10, w10, w12 add w10, w10, w13 ; Round 10 - mov w14, v5.s[0] + mov w14, V5.S[0] ror w12, w6, #6 eor w13, w7, w8 eor w12, w12, w6, ror 11 @@ -446,50 +446,50 @@ L_sha256_len_neon_start add w9, w9, w12 add w9, w9, w13 ; Round 11 - mov w14, v5.s[1] + mov w14, V5.S[1] ; Calc new W[10]-W[11] - ext v10.8b, v5.8b, v6.8b, #4 + ext8 V10.8B, V5.8B, V6.8B, #4 ror w12, w5, #6 - shl v8.2s, v4.2s, #15 + shl V8.2S, V4.2S, #15 eor w13, w6, w7 - sri v8.2s, v4.2s, #17 + sri V8.2S, V4.2S, #17 eor w12, w12, w5, ror 11 - shl v9.2s, v4.2s, #13 + shl V9.2S, V4.2S, #13 and w13, w13, w5 - sri v9.2s, v4.2s, #19 + sri V9.2S, V4.2S, #19 eor w12, w12, w5, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w7 - ushr v8.2s, v4.2s, #10 + ushr V8.2S, V4.2S, #10 add w8, w8, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w8, w8, w13 - add v5.2s, v5.2s, v9.2s + add V5.2S, V5.2S, V9.2S ldr w12, [x3, #44] - ext v11.8b, v1.8b, v2.8b, #4 + ext8 V11.8B, V1.8B, V2.8B, #4 add w8, w8, w14 - add v5.2s, v5.2s, v11.2s + add V5.2S, V5.2S, V11.2S add w8, w8, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w4, w4, w8 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w9, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w9, w10 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w9, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w10, w11 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w9, ror 22 - add v5.2s, v5.2s, v9.2s + add V5.2S, V5.2S, V9.2S eor w13, w13, w10 add w8, w8, w12 add w8, w8, w13 ; Round 12 - mov w14, v6.s[0] + mov w14, V6.S[0] ror w12, w4, #6 eor w13, w5, w6 eor w12, w12, w4, ror 11 @@ -512,50 +512,50 @@ L_sha256_len_neon_start add w7, w7, w12 add w7, w7, w13 ; Round 13 - mov w14, v6.s[1] + mov w14, V6.S[1] ; Calc new W[12]-W[13] - ext v10.8b, v6.8b, v7.8b, #4 + ext8 V10.8B, V6.8B, V7.8B, #4 ror w12, w11, #6 - shl v8.2s, v5.2s, #15 + shl V8.2S, V5.2S, #15 eor w13, w4, w5 - sri v8.2s, v5.2s, #17 + sri V8.2S, V5.2S, #17 eor w12, w12, w11, ror 11 - shl v9.2s, v5.2s, #13 + shl V9.2S, V5.2S, #13 and w13, w13, w11 - sri v9.2s, v5.2s, #19 + sri V9.2S, V5.2S, #19 eor w12, w12, w11, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w5 - ushr v8.2s, v5.2s, #10 + ushr V8.2S, V5.2S, #10 add w6, w6, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w6, w6, w13 - add v6.2s, v6.2s, v9.2s + add V6.2S, V6.2S, V9.2S ldr w12, [x3, #52] - ext v11.8b, v2.8b, v3.8b, #4 + ext8 V11.8B, V2.8B, V3.8B, #4 add w6, w6, w14 - add v6.2s, v6.2s, v11.2s + add V6.2S, V6.2S, V11.2S add w6, w6, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w10, w10, w6 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w7, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w7, w8 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w7, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w8, w9 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w7, ror 22 - add v6.2s, v6.2s, v9.2s + add V6.2S, V6.2S, V9.2S eor w13, w13, w8 add w6, w6, w12 add w6, w6, w13 ; Round 14 - mov w14, v7.s[0] + mov w14, V7.S[0] ror w12, w10, #6 eor w13, w11, w4 eor w12, w12, w10, ror 11 @@ -578,45 +578,45 @@ L_sha256_len_neon_start add w5, w5, w12 add w5, w5, w13 ; Round 15 - mov w14, v7.s[1] + mov w14, V7.S[1] ; Calc new W[14]-W[15] - ext v10.8b, v7.8b, v0.8b, #4 + ext8 V10.8B, V7.8B, V0.8B, #4 ror w12, w9, #6 - shl v8.2s, v6.2s, #15 + shl V8.2S, V6.2S, #15 eor w13, w10, w11 - sri v8.2s, v6.2s, #17 + sri V8.2S, V6.2S, #17 eor w12, w12, w9, ror 11 - shl v9.2s, v6.2s, #13 + shl V9.2S, V6.2S, #13 and w13, w13, w9 - sri v9.2s, v6.2s, #19 + sri V9.2S, V6.2S, #19 eor w12, w12, w9, ror 25 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w13, w13, w11 - ushr v8.2s, v6.2s, #10 + ushr V8.2S, V6.2S, #10 add w4, w4, w12 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B add w4, w4, w13 - add v7.2s, v7.2s, v9.2s + add V7.2S, V7.2S, V9.2S ldr w12, [x3, #60] - ext v11.8b, v3.8b, v4.8b, #4 + ext8 V11.8B, V3.8B, V4.8B, #4 add w4, w4, w14 - add v7.2s, v7.2s, v11.2s + add V7.2S, V7.2S, V11.2S add w4, w4, w12 - shl v8.2s, v10.2s, #25 + shl V8.2S, V10.2S, #25 add w8, w8, w4 - sri v8.2s, v10.2s, #7 + sri V8.2S, V10.2S, #7 ror w12, w5, #2 - shl v9.2s, v10.2s, #14 + shl V9.2S, V10.2S, #14 eor w13, w5, w6 - sri v9.2s, v10.2s, #18 + sri V9.2S, V10.2S, #18 eor w12, w12, w5, ror 13 - eor v9.8b, v9.8b, v8.8b + eor V9.8B, V9.8B, V8.8B eor w14, w6, w7 - ushr v10.2s, v10.2s, #3 + ushr V10.2S, V10.2S, #3 and w13, w13, w14 - eor v9.8b, v9.8b, v10.8b + eor V9.8B, V9.8B, V10.8B eor w12, w12, w5, ror 22 - add v7.2s, v7.2s, v9.2s + add V7.2S, V7.2S, V9.2S eor w13, w13, w6 add w4, w4, w12 add w4, w4, w13 @@ -624,7 +624,7 @@ L_sha256_len_neon_start subs x24, x24, #1 bne L_sha256_len_neon_start ; Round 0 - mov w14, v0.s[0] + mov w14, V0.S[0] ror w12, w8, #6 eor w13, w9, w10 eor w12, w12, w8, ror 11 @@ -647,7 +647,7 @@ L_sha256_len_neon_start add w11, w11, w12 add w11, w11, w13 ; Round 1 - mov w14, v0.s[1] + mov w14, V0.S[1] ror w12, w7, #6 eor w13, w8, w9 eor w12, w12, w7, ror 11 @@ -670,7 +670,7 @@ L_sha256_len_neon_start add w10, w10, w12 add w10, w10, w13 ; Round 2 - mov w14, v1.s[0] + mov w14, V1.S[0] ror w12, w6, #6 eor w13, w7, w8 eor w12, w12, w6, ror 11 @@ -693,7 +693,7 @@ L_sha256_len_neon_start add w9, w9, w12 add w9, w9, w13 ; Round 3 - mov w14, v1.s[1] + mov w14, V1.S[1] ror w12, w5, #6 eor w13, w6, w7 eor w12, w12, w5, ror 11 @@ -716,7 +716,7 @@ L_sha256_len_neon_start add w8, w8, w12 add w8, w8, w13 ; Round 4 - mov w14, v2.s[0] + mov w14, V2.S[0] ror w12, w4, #6 eor w13, w5, w6 eor w12, w12, w4, ror 11 @@ -739,7 +739,7 @@ L_sha256_len_neon_start add w7, w7, w12 add w7, w7, w13 ; Round 5 - mov w14, v2.s[1] + mov w14, V2.S[1] ror w12, w11, #6 eor w13, w4, w5 eor w12, w12, w11, ror 11 @@ -762,7 +762,7 @@ L_sha256_len_neon_start add w6, w6, w12 add w6, w6, w13 ; Round 6 - mov w14, v3.s[0] + mov w14, V3.S[0] ror w12, w10, #6 eor w13, w11, w4 eor w12, w12, w10, ror 11 @@ -785,7 +785,7 @@ L_sha256_len_neon_start add w5, w5, w12 add w5, w5, w13 ; Round 7 - mov w14, v3.s[1] + mov w14, V3.S[1] ror w12, w9, #6 eor w13, w10, w11 eor w12, w12, w9, ror 11 @@ -808,7 +808,7 @@ L_sha256_len_neon_start add w4, w4, w12 add w4, w4, w13 ; Round 8 - mov w14, v4.s[0] + mov w14, V4.S[0] ror w12, w8, #6 eor w13, w9, w10 eor w12, w12, w8, ror 11 @@ -831,7 +831,7 @@ L_sha256_len_neon_start add w11, w11, w12 add w11, w11, w13 ; Round 9 - mov w14, v4.s[1] + mov w14, V4.S[1] ror w12, w7, #6 eor w13, w8, w9 eor w12, w12, w7, ror 11 @@ -854,7 +854,7 @@ L_sha256_len_neon_start add w10, w10, w12 add w10, w10, w13 ; Round 10 - mov w14, v5.s[0] + mov w14, V5.S[0] ror w12, w6, #6 eor w13, w7, w8 eor w12, w12, w6, ror 11 @@ -877,7 +877,7 @@ L_sha256_len_neon_start add w9, w9, w12 add w9, w9, w13 ; Round 11 - mov w14, v5.s[1] + mov w14, V5.S[1] ror w12, w5, #6 eor w13, w6, w7 eor w12, w12, w5, ror 11 @@ -900,7 +900,7 @@ L_sha256_len_neon_start add w8, w8, w12 add w8, w8, w13 ; Round 12 - mov w14, v6.s[0] + mov w14, V6.S[0] ror w12, w4, #6 eor w13, w5, w6 eor w12, w12, w4, ror 11 @@ -923,7 +923,7 @@ L_sha256_len_neon_start add w7, w7, w12 add w7, w7, w13 ; Round 13 - mov w14, v6.s[1] + mov w14, V6.S[1] ror w12, w11, #6 eor w13, w4, w5 eor w12, w12, w11, ror 11 @@ -946,7 +946,7 @@ L_sha256_len_neon_start add w6, w6, w12 add w6, w6, w13 ; Round 14 - mov w14, v7.s[0] + mov w14, V7.S[0] ror w12, w10, #6 eor w13, w11, w4 eor w12, w12, w10, ror 11 @@ -969,7 +969,7 @@ L_sha256_len_neon_start add w5, w5, w12 add w5, w5, w13 ; Round 15 - mov w14, v7.s[1] + mov w14, V7.S[1] ror w12, w9, #6 eor w13, w10, w11 eor w12, w12, w9, ror 11 @@ -1014,13 +1014,13 @@ L_sha256_len_neon_start ldp x20, x21, [x29, #40] ldp x22, x23, [x29, #56] ldr x24, [x29, #72] - ldp d8, d9, [x29, #80] - ldp d10, d11, [x29, #96] + ldp D8, D9, [x29, #80] + ldp D10, D11, [x29, #96] ldp x29, x30, [sp], #0x70 ret ENDP IF :LNOT::DEF:WOLFSSL_ARMASM_NO_HW_CRYPTO - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 8 L_SHA256_trans_crypto_len_k DCD 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 @@ -1045,146 +1045,146 @@ L_SHA256_trans_crypto_len_k Transform_Sha256_Len_crypto PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x3, L_SHA256_trans_crypto_len_k add x3, x3, L_SHA256_trans_crypto_len_k ; Load K into vector registers - ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x3], #0x40 - ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x3], #0x40 - ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x3], #0x40 - ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x3], #0x40 + ld1 {V8.4S, V9.4S, V10.4S, V11.4S}, [x3], #0x40 + ld1 {V12.4S, V13.4S, V14.4S, V15.4S}, [x3], #0x40 + ld1 {V16.4S, V17.4S, V18.4S, V19.4S}, [x3], #0x40 + ld1 {V20.4S, V21.4S, V22.4S, V23.4S}, [x3], #0x40 ; Load digest into working vars - ld1 {v0.4s, v1.4s}, [x0] + ld1 {V0.4S, V1.4S}, [x0] ; Start of loop processing a block L_sha256_len_crypto_begin ; Load W - ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x1], #0x40 - rev32 v4.16b, v4.16b - rev32 v5.16b, v5.16b - rev32 v6.16b, v6.16b - rev32 v7.16b, v7.16b + ld1 {V4.4S, V5.4S, V6.4S, V7.4S}, [x1], #0x40 + rev32 V4.16B, V4.16B + rev32 V5.16B, V5.16B + rev32 V6.16B, V6.16B + rev32 V7.16B, V7.16B ; Copy digest to add in at end - mov v2.16b, v0.16b - mov v3.16b, v1.16b + mov V2.16B, V0.16B + mov V3.16B, V1.16B ; Start 16 rounds ; Round 1 - add v24.4s, v4.4s, v8.4s - mov v25.16b, v0.16b - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + add V24.4S, V4.4S, V8.4S + mov V25.16B, V0.16B + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 2 - sha256su0 v4.4s, v5.4s - add v24.4s, v5.4s, v9.4s - mov v25.16b, v0.16b - sha256su1 v4.4s, v6.4s, v7.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V4.4S, V5.4S + add V24.4S, V5.4S, V9.4S + mov V25.16B, V0.16B + sha256su1 V4.4S, V6.4S, V7.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 3 - sha256su0 v5.4s, v6.4s - add v24.4s, v6.4s, v10.4s - mov v25.16b, v0.16b - sha256su1 v5.4s, v7.4s, v4.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V5.4S, V6.4S + add V24.4S, V6.4S, V10.4S + mov V25.16B, V0.16B + sha256su1 V5.4S, V7.4S, V4.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 4 - sha256su0 v6.4s, v7.4s - add v24.4s, v7.4s, v11.4s - mov v25.16b, v0.16b - sha256su1 v6.4s, v4.4s, v5.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V6.4S, V7.4S + add V24.4S, V7.4S, V11.4S + mov V25.16B, V0.16B + sha256su1 V6.4S, V4.4S, V5.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 5 - sha256su0 v7.4s, v4.4s - add v24.4s, v4.4s, v12.4s - mov v25.16b, v0.16b - sha256su1 v7.4s, v5.4s, v6.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V7.4S, V4.4S + add V24.4S, V4.4S, V12.4S + mov V25.16B, V0.16B + sha256su1 V7.4S, V5.4S, V6.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 6 - sha256su0 v4.4s, v5.4s - add v24.4s, v5.4s, v13.4s - mov v25.16b, v0.16b - sha256su1 v4.4s, v6.4s, v7.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V4.4S, V5.4S + add V24.4S, V5.4S, V13.4S + mov V25.16B, V0.16B + sha256su1 V4.4S, V6.4S, V7.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 7 - sha256su0 v5.4s, v6.4s - add v24.4s, v6.4s, v14.4s - mov v25.16b, v0.16b - sha256su1 v5.4s, v7.4s, v4.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V5.4S, V6.4S + add V24.4S, V6.4S, V14.4S + mov V25.16B, V0.16B + sha256su1 V5.4S, V7.4S, V4.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 8 - sha256su0 v6.4s, v7.4s - add v24.4s, v7.4s, v15.4s - mov v25.16b, v0.16b - sha256su1 v6.4s, v4.4s, v5.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V6.4S, V7.4S + add V24.4S, V7.4S, V15.4S + mov V25.16B, V0.16B + sha256su1 V6.4S, V4.4S, V5.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 9 - sha256su0 v7.4s, v4.4s - add v24.4s, v4.4s, v16.4s - mov v25.16b, v0.16b - sha256su1 v7.4s, v5.4s, v6.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V7.4S, V4.4S + add V24.4S, V4.4S, V16.4S + mov V25.16B, V0.16B + sha256su1 V7.4S, V5.4S, V6.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 10 - sha256su0 v4.4s, v5.4s - add v24.4s, v5.4s, v17.4s - mov v25.16b, v0.16b - sha256su1 v4.4s, v6.4s, v7.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V4.4S, V5.4S + add V24.4S, V5.4S, V17.4S + mov V25.16B, V0.16B + sha256su1 V4.4S, V6.4S, V7.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 11 - sha256su0 v5.4s, v6.4s - add v24.4s, v6.4s, v18.4s - mov v25.16b, v0.16b - sha256su1 v5.4s, v7.4s, v4.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V5.4S, V6.4S + add V24.4S, V6.4S, V18.4S + mov V25.16B, V0.16B + sha256su1 V5.4S, V7.4S, V4.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 12 - sha256su0 v6.4s, v7.4s - add v24.4s, v7.4s, v19.4s - mov v25.16b, v0.16b - sha256su1 v6.4s, v4.4s, v5.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V6.4S, V7.4S + add V24.4S, V7.4S, V19.4S + mov V25.16B, V0.16B + sha256su1 V6.4S, V4.4S, V5.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 13 - sha256su0 v7.4s, v4.4s - add v24.4s, v4.4s, v20.4s - mov v25.16b, v0.16b - sha256su1 v7.4s, v5.4s, v6.4s - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + sha256su0 V7.4S, V4.4S + add V24.4S, V4.4S, V20.4S + mov V25.16B, V0.16B + sha256su1 V7.4S, V5.4S, V6.4S + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 14 - add v24.4s, v5.4s, v21.4s - mov v25.16b, v0.16b - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + add V24.4S, V5.4S, V21.4S + mov V25.16B, V0.16B + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 15 - add v24.4s, v6.4s, v22.4s - mov v25.16b, v0.16b - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + add V24.4S, V6.4S, V22.4S + mov V25.16B, V0.16B + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Round 16 - add v24.4s, v7.4s, v23.4s - mov v25.16b, v0.16b - sha256h q0, q1, v24.4s - sha256h2 q1, q25, v24.4s + add V24.4S, V7.4S, V23.4S + mov V25.16B, V0.16B + sha256h Q0, Q1, V24.4S + sha256h2 Q1, Q25, V24.4S ; Done 16 rounds - add v0.4s, v0.4s, v2.4s - add v1.4s, v1.4s, v3.4s + add V0.4S, V0.4S, V2.4S + add V1.4S, V1.4S, V3.4S subs w2, w2, #0x40 bne L_sha256_len_crypto_begin ; Store digest back - st1 {v0.4s, v1.4s}, [x0] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st1 {V0.4S, V1.4S}, [x0] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP diff --git a/wolfcrypt/src/port/arm/armv8-sha3-asm.asm b/wolfcrypt/src/port/arm/armv8-sha3-asm.asm index fcd1929a36..f131e69e6d 100644 --- a/wolfcrypt/src/port/arm/armv8-sha3-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-sha3-asm.asm @@ -26,7 +26,7 @@ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-sha3-asm.asm IF :DEF:WOLFSSL_SHA3 IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA3 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA3_transform_crypto_r DCQ 0x0000000000000001, 0x0000000000008082 @@ -47,120 +47,120 @@ L_SHA3_transform_crypto_r BlockSha3_crypto PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x1, L_SHA3_transform_crypto_r add x1, x1, L_SHA3_transform_crypto_r ; .arch_extension sha3 - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.1d}, [x0] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.1D}, [x0] sub x0, x0, #0xc0 mov x2, #24 ; Start of 24 rounds L_sha3_crypto_begin ; Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b - eor3 v27.16b, v1.16b, v6.16b, v11.16b - eor3 v28.16b, v2.16b, v7.16b, v12.16b - eor3 v29.16b, v3.16b, v8.16b, v13.16b - eor3 v30.16b, v4.16b, v9.16b, v14.16b - eor3 v31.16b, v31.16b, v15.16b, v20.16b - eor3 v27.16b, v27.16b, v16.16b, v21.16b - eor3 v28.16b, v28.16b, v17.16b, v22.16b - eor3 v29.16b, v29.16b, v18.16b, v23.16b - eor3 v30.16b, v30.16b, v19.16b, v24.16b - rax1 v25.2d, v30.2d, v27.2d - rax1 v26.2d, v31.2d, v28.2d - rax1 v27.2d, v27.2d, v29.2d - rax1 v28.2d, v28.2d, v30.2d - rax1 v29.2d, v29.2d, v31.2d - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 - xar v1.2d, v6.2d, v26.2d, #20 - xar v6.2d, v9.2d, v29.2d, #44 - xar v9.2d, v22.2d, v27.2d, #3 - xar v22.2d, v14.2d, v29.2d, #25 - xar v14.2d, v20.2d, v25.2d, #46 - xar v20.2d, v2.2d, v27.2d, #2 - xar v2.2d, v12.2d, v27.2d, #21 - xar v12.2d, v13.2d, v28.2d, #39 - xar v13.2d, v19.2d, v29.2d, #56 - xar v19.2d, v23.2d, v28.2d, #8 - xar v23.2d, v15.2d, v25.2d, #23 - xar v15.2d, v4.2d, v29.2d, #37 - xar v4.2d, v24.2d, v29.2d, #50 - xar v24.2d, v21.2d, v26.2d, #62 - xar v21.2d, v8.2d, v28.2d, #9 - xar v8.2d, v16.2d, v26.2d, #19 - xar v16.2d, v5.2d, v25.2d, #28 - xar v5.2d, v3.2d, v28.2d, #36 - xar v3.2d, v18.2d, v28.2d, #43 - xar v18.2d, v17.2d, v27.2d, #49 - xar v17.2d, v11.2d, v26.2d, #54 - xar v11.2d, v7.2d, v27.2d, #58 - xar v7.2d, v10.2d, v25.2d, #61 + eor3 V31.16B, V0.16B, V5.16B, V10.16B + eor3 V27.16B, V1.16B, V6.16B, V11.16B + eor3 V28.16B, V2.16B, V7.16B, V12.16B + eor3 V29.16B, V3.16B, V8.16B, V13.16B + eor3 V30.16B, V4.16B, V9.16B, V14.16B + eor3 V31.16B, V31.16B, V15.16B, V20.16B + eor3 V27.16B, V27.16B, V16.16B, V21.16B + eor3 V28.16B, V28.16B, V17.16B, V22.16B + eor3 V29.16B, V29.16B, V18.16B, V23.16B + eor3 V30.16B, V30.16B, V19.16B, V24.16B + rax1 V25.2D, V30.2D, V27.2D + rax1 V26.2D, V31.2D, V28.2D + rax1 V27.2D, V27.2D, V29.2D + rax1 V28.2D, V28.2D, V30.2D + rax1 V29.2D, V29.2D, V31.2D + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 + xar V1.2D, V6.2D, V26.2D, #20 + xar V6.2D, V9.2D, V29.2D, #44 + xar V9.2D, V22.2D, V27.2D, #3 + xar V22.2D, V14.2D, V29.2D, #25 + xar V14.2D, V20.2D, V25.2D, #46 + xar V20.2D, V2.2D, V27.2D, #2 + xar V2.2D, V12.2D, V27.2D, #21 + xar V12.2D, V13.2D, V28.2D, #39 + xar V13.2D, V19.2D, V29.2D, #56 + xar V19.2D, V23.2D, V28.2D, #8 + xar V23.2D, V15.2D, V25.2D, #23 + xar V15.2D, V4.2D, V29.2D, #37 + xar V4.2D, V24.2D, V29.2D, #50 + xar V24.2D, V21.2D, V26.2D, #62 + xar V21.2D, V8.2D, V28.2D, #9 + xar V8.2D, V16.2D, V26.2D, #19 + xar V16.2D, V5.2D, V25.2D, #28 + xar V5.2D, V3.2D, V28.2D, #36 + xar V3.2D, V18.2D, V28.2D, #43 + xar V18.2D, V17.2D, V27.2D, #49 + xar V17.2D, V11.2D, V26.2D, #54 + xar V11.2D, V7.2D, V27.2D, #58 + xar V7.2D, V10.2D, V25.2D, #61 ; Row Mix - mov v25.16b, v0.16b - mov v26.16b, v1.16b - bcax v0.16b, v25.16b, v2.16b, v26.16b - bcax v1.16b, v26.16b, v3.16b, v2.16b - bcax v2.16b, v2.16b, v4.16b, v3.16b - bcax v3.16b, v3.16b, v25.16b, v4.16b - bcax v4.16b, v4.16b, v26.16b, v25.16b - mov v25.16b, v5.16b - mov v26.16b, v6.16b - bcax v5.16b, v25.16b, v7.16b, v26.16b - bcax v6.16b, v26.16b, v8.16b, v7.16b - bcax v7.16b, v7.16b, v9.16b, v8.16b - bcax v8.16b, v8.16b, v25.16b, v9.16b - bcax v9.16b, v9.16b, v26.16b, v25.16b - mov v26.16b, v11.16b - bcax v10.16b, v30.16b, v12.16b, v26.16b - bcax v11.16b, v26.16b, v13.16b, v12.16b - bcax v12.16b, v12.16b, v14.16b, v13.16b - bcax v13.16b, v13.16b, v30.16b, v14.16b - bcax v14.16b, v14.16b, v26.16b, v30.16b - mov v25.16b, v15.16b - mov v26.16b, v16.16b - bcax v15.16b, v25.16b, v17.16b, v26.16b - bcax v16.16b, v26.16b, v18.16b, v17.16b - bcax v17.16b, v17.16b, v19.16b, v18.16b - bcax v18.16b, v18.16b, v25.16b, v19.16b - bcax v19.16b, v19.16b, v26.16b, v25.16b - mov v25.16b, v20.16b - mov v26.16b, v21.16b - bcax v20.16b, v25.16b, v22.16b, v26.16b - bcax v21.16b, v26.16b, v23.16b, v22.16b - bcax v22.16b, v22.16b, v24.16b, v23.16b - bcax v23.16b, v23.16b, v25.16b, v24.16b - bcax v24.16b, v24.16b, v26.16b, v25.16b - ld1r {v30.2d}, [x1], #8 + mov V25.16B, V0.16B + mov V26.16B, V1.16B + bcax V0.16B, V25.16B, V2.16B, V26.16B + bcax V1.16B, V26.16B, V3.16B, V2.16B + bcax V2.16B, V2.16B, V4.16B, V3.16B + bcax V3.16B, V3.16B, V25.16B, V4.16B + bcax V4.16B, V4.16B, V26.16B, V25.16B + mov V25.16B, V5.16B + mov V26.16B, V6.16B + bcax V5.16B, V25.16B, V7.16B, V26.16B + bcax V6.16B, V26.16B, V8.16B, V7.16B + bcax V7.16B, V7.16B, V9.16B, V8.16B + bcax V8.16B, V8.16B, V25.16B, V9.16B + bcax V9.16B, V9.16B, V26.16B, V25.16B + mov V26.16B, V11.16B + bcax V10.16B, V30.16B, V12.16B, V26.16B + bcax V11.16B, V26.16B, V13.16B, V12.16B + bcax V12.16B, V12.16B, V14.16B, V13.16B + bcax V13.16B, V13.16B, V30.16B, V14.16B + bcax V14.16B, V14.16B, V26.16B, V30.16B + mov V25.16B, V15.16B + mov V26.16B, V16.16B + bcax V15.16B, V25.16B, V17.16B, V26.16B + bcax V16.16B, V26.16B, V18.16B, V17.16B + bcax V17.16B, V17.16B, V19.16B, V18.16B + bcax V18.16B, V18.16B, V25.16B, V19.16B + bcax V19.16B, V19.16B, V26.16B, V25.16B + mov V25.16B, V20.16B + mov V26.16B, V21.16B + bcax V20.16B, V25.16B, V22.16B, V26.16B + bcax V21.16B, V26.16B, V23.16B, V22.16B + bcax V22.16B, V22.16B, V24.16B, V23.16B + bcax V23.16B, V23.16B, V25.16B, V24.16B + bcax V24.16B, V24.16B, V26.16B, V25.16B + ld1r {V30.2D}, [x1], #8 subs x2, x2, #1 - eor v0.16b, v0.16b, v30.16b + eor V0.16B, V0.16B, V30.16B bne L_sha3_crypto_begin - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.1d}, [x0] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.1D}, [x0] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP ENDIF - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA3_transform_base_r DCQ 0x0000000000000001, 0x0000000000008082 diff --git a/wolfcrypt/src/port/arm/armv8-sha512-asm.asm b/wolfcrypt/src/port/arm/armv8-sha512-asm.asm index 4c333808e6..c257f9183a 100644 --- a/wolfcrypt/src/port/arm/armv8-sha512-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-sha512-asm.asm @@ -25,7 +25,7 @@ ; ruby ./sha2/sha512.rb arm64 \ ; ../wolfssl/wolfcrypt/src/port/arm/armv8-sha512-asm.asm IF :DEF:WOLFSSL_SHA512 :LOR: :DEF:WOLFSSL_SHA384 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA512_transform_neon_len_k DCQ 0x428a2f98d728ae22, 0x7137449123ef65cd @@ -68,7 +68,7 @@ L_SHA512_transform_neon_len_k DCQ 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c DCQ 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a DCQ 0x5fcb6fab3ad6faec, 0x6c44198c4a475817 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA512_transform_neon_len_r8 DCQ 0x0007060504030201, 0x080f0e0d0c0b0a09 @@ -83,13 +83,13 @@ Transform_Sha512_Len_neon PROC stp x22, x23, [x29, #48] stp x24, x25, [x29, #64] stp x26, x27, [x29, #80] - stp d8, d9, [x29, #96] - stp d10, d11, [x29, #112] + stp D8, D9, [x29, #96] + stp D10, D11, [x29, #112] adrp x3, L_SHA512_transform_neon_len_k add x3, x3, L_SHA512_transform_neon_len_k adrp x27, L_SHA512_transform_neon_len_r8 add x27, x27, L_SHA512_transform_neon_len_r8 - ld1 {v11.16b}, [x27] + ld1 {V11.16B}, [x27] ; Load digest into working vars ldp x4, x5, [x0] ldp x6, x7, [x0, #16] @@ -99,31 +99,31 @@ Transform_Sha512_Len_neon PROC L_sha512_len_neon_begin ; Load W ; Copy digest to add in at end - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 mov x19, x4 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 mov x20, x5 - rev64 v0.16b, v0.16b + rev64 V0.16B, V0.16B mov x21, x6 - rev64 v1.16b, v1.16b + rev64 V1.16B, V1.16B mov x22, x7 - rev64 v2.16b, v2.16b + rev64 V2.16B, V2.16B mov x23, x8 - rev64 v3.16b, v3.16b + rev64 V3.16B, V3.16B mov x24, x9 - rev64 v4.16b, v4.16b + rev64 V4.16B, V4.16B mov x25, x10 - rev64 v5.16b, v5.16b + rev64 V5.16B, V5.16B mov x26, x11 - rev64 v6.16b, v6.16b - rev64 v7.16b, v7.16b + rev64 V6.16B, V6.16B + rev64 V7.16B, V7.16B ; Pre-calc: b ^ c eor x16, x5, x6 mov x27, #4 ; Start of 16 rounds L_sha512_len_neon_start ; Round 0 - mov x13, v0.d[0] + mov x13, V0.D[0] ldr x15, [x3], #8 ror x12, x8, #14 ror x14, x4, #28 @@ -145,47 +145,47 @@ L_sha512_len_neon_start add x7, x7, x11 add x11, x11, x14 ; Round 1 - mov x13, v0.d[1] + mov x13, V0.D[1] ldr x15, [x3], #8 - ext v10.16b, v0.16b, v1.16b, #8 + ext8 V10.16B, V0.16B, V1.16B, #8 ror x12, x7, #14 - shl v8.2d, v7.2d, #45 + shl V8.2D, V7.2D, #45 ror x14, x11, #28 - sri v8.2d, v7.2d, #19 + sri V8.2D, V7.2D, #19 eor x12, x12, x7, ror 18 - shl v9.2d, v7.2d, #3 + shl V9.2D, V7.2D, #3 eor x14, x14, x11, ror 34 - sri v9.2d, v7.2d, #61 + sri V9.2D, V7.2D, #61 eor x12, x12, x7, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x11, ror 39 - ushr v8.2d, v7.2d, #6 + ushr V8.2D, V7.2D, #6 add x10, x10, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x11, x4 - add v0.2d, v0.2d, v9.2d + add V0.2D, V0.2D, V9.2D eor x12, x8, x9 - ext v9.16b, v4.16b, v5.16b, #8 + ext8 V9.16B, V4.16B, V5.16B, #8 and x17, x16, x17 - add v0.2d, v0.2d, v9.2d + add V0.2D, V0.2D, V9.2D and x12, x12, x7 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x10, x10, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x9 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x10, x10, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x4 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x10, x10, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v0.2d, v0.2d, v9.2d + add V0.2D, V0.2D, V9.2D add x6, x6, x10 add x10, x10, x14 ; Round 2 - mov x13, v1.d[0] + mov x13, V1.D[0] ldr x15, [x3], #8 ror x12, x6, #14 ror x14, x10, #28 @@ -207,47 +207,47 @@ L_sha512_len_neon_start add x5, x5, x9 add x9, x9, x14 ; Round 3 - mov x13, v1.d[1] + mov x13, V1.D[1] ldr x15, [x3], #8 - ext v10.16b, v1.16b, v2.16b, #8 + ext8 V10.16B, V1.16B, V2.16B, #8 ror x12, x5, #14 - shl v8.2d, v0.2d, #45 + shl V8.2D, V0.2D, #45 ror x14, x9, #28 - sri v8.2d, v0.2d, #19 + sri V8.2D, V0.2D, #19 eor x12, x12, x5, ror 18 - shl v9.2d, v0.2d, #3 + shl V9.2D, V0.2D, #3 eor x14, x14, x9, ror 34 - sri v9.2d, v0.2d, #61 + sri V9.2D, V0.2D, #61 eor x12, x12, x5, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x9, ror 39 - ushr v8.2d, v0.2d, #6 + ushr V8.2D, V0.2D, #6 add x8, x8, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x9, x10 - add v1.2d, v1.2d, v9.2d + add V1.2D, V1.2D, V9.2D eor x12, x6, x7 - ext v9.16b, v5.16b, v6.16b, #8 + ext8 V9.16B, V5.16B, V6.16B, #8 and x17, x16, x17 - add v1.2d, v1.2d, v9.2d + add V1.2D, V1.2D, V9.2D and x12, x12, x5 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x8, x8, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x7 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x8, x8, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x10 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x8, x8, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v1.2d, v1.2d, v9.2d + add V1.2D, V1.2D, V9.2D add x4, x4, x8 add x8, x8, x14 ; Round 4 - mov x13, v2.d[0] + mov x13, V2.D[0] ldr x15, [x3], #8 ror x12, x4, #14 ror x14, x8, #28 @@ -269,47 +269,47 @@ L_sha512_len_neon_start add x11, x11, x7 add x7, x7, x14 ; Round 5 - mov x13, v2.d[1] + mov x13, V2.D[1] ldr x15, [x3], #8 - ext v10.16b, v2.16b, v3.16b, #8 + ext8 V10.16B, V2.16B, V3.16B, #8 ror x12, x11, #14 - shl v8.2d, v1.2d, #45 + shl V8.2D, V1.2D, #45 ror x14, x7, #28 - sri v8.2d, v1.2d, #19 + sri V8.2D, V1.2D, #19 eor x12, x12, x11, ror 18 - shl v9.2d, v1.2d, #3 + shl V9.2D, V1.2D, #3 eor x14, x14, x7, ror 34 - sri v9.2d, v1.2d, #61 + sri V9.2D, V1.2D, #61 eor x12, x12, x11, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x7, ror 39 - ushr v8.2d, v1.2d, #6 + ushr V8.2D, V1.2D, #6 add x6, x6, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x7, x8 - add v2.2d, v2.2d, v9.2d + add V2.2D, V2.2D, V9.2D eor x12, x4, x5 - ext v9.16b, v6.16b, v7.16b, #8 + ext8 V9.16B, V6.16B, V7.16B, #8 and x17, x16, x17 - add v2.2d, v2.2d, v9.2d + add V2.2D, V2.2D, V9.2D and x12, x12, x11 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x6, x6, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x5 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x6, x6, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x8 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x6, x6, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v2.2d, v2.2d, v9.2d + add V2.2D, V2.2D, V9.2D add x10, x10, x6 add x6, x6, x14 ; Round 6 - mov x13, v3.d[0] + mov x13, V3.D[0] ldr x15, [x3], #8 ror x12, x10, #14 ror x14, x6, #28 @@ -331,47 +331,47 @@ L_sha512_len_neon_start add x9, x9, x5 add x5, x5, x14 ; Round 7 - mov x13, v3.d[1] + mov x13, V3.D[1] ldr x15, [x3], #8 - ext v10.16b, v3.16b, v4.16b, #8 + ext8 V10.16B, V3.16B, V4.16B, #8 ror x12, x9, #14 - shl v8.2d, v2.2d, #45 + shl V8.2D, V2.2D, #45 ror x14, x5, #28 - sri v8.2d, v2.2d, #19 + sri V8.2D, V2.2D, #19 eor x12, x12, x9, ror 18 - shl v9.2d, v2.2d, #3 + shl V9.2D, V2.2D, #3 eor x14, x14, x5, ror 34 - sri v9.2d, v2.2d, #61 + sri V9.2D, V2.2D, #61 eor x12, x12, x9, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x5, ror 39 - ushr v8.2d, v2.2d, #6 + ushr V8.2D, V2.2D, #6 add x4, x4, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x5, x6 - add v3.2d, v3.2d, v9.2d + add V3.2D, V3.2D, V9.2D eor x12, x10, x11 - ext v9.16b, v7.16b, v0.16b, #8 + ext8 V9.16B, V7.16B, V0.16B, #8 and x17, x16, x17 - add v3.2d, v3.2d, v9.2d + add V3.2D, V3.2D, V9.2D and x12, x12, x9 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x4, x4, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x11 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x4, x4, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x6 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x4, x4, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v3.2d, v3.2d, v9.2d + add V3.2D, V3.2D, V9.2D add x8, x8, x4 add x4, x4, x14 ; Round 8 - mov x13, v4.d[0] + mov x13, V4.D[0] ldr x15, [x3], #8 ror x12, x8, #14 ror x14, x4, #28 @@ -393,47 +393,47 @@ L_sha512_len_neon_start add x7, x7, x11 add x11, x11, x14 ; Round 9 - mov x13, v4.d[1] + mov x13, V4.D[1] ldr x15, [x3], #8 - ext v10.16b, v4.16b, v5.16b, #8 + ext8 V10.16B, V4.16B, V5.16B, #8 ror x12, x7, #14 - shl v8.2d, v3.2d, #45 + shl V8.2D, V3.2D, #45 ror x14, x11, #28 - sri v8.2d, v3.2d, #19 + sri V8.2D, V3.2D, #19 eor x12, x12, x7, ror 18 - shl v9.2d, v3.2d, #3 + shl V9.2D, V3.2D, #3 eor x14, x14, x11, ror 34 - sri v9.2d, v3.2d, #61 + sri V9.2D, V3.2D, #61 eor x12, x12, x7, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x11, ror 39 - ushr v8.2d, v3.2d, #6 + ushr V8.2D, V3.2D, #6 add x10, x10, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x11, x4 - add v4.2d, v4.2d, v9.2d + add V4.2D, V4.2D, V9.2D eor x12, x8, x9 - ext v9.16b, v0.16b, v1.16b, #8 + ext8 V9.16B, V0.16B, V1.16B, #8 and x17, x16, x17 - add v4.2d, v4.2d, v9.2d + add V4.2D, V4.2D, V9.2D and x12, x12, x7 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x10, x10, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x9 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x10, x10, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x4 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x10, x10, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v4.2d, v4.2d, v9.2d + add V4.2D, V4.2D, V9.2D add x6, x6, x10 add x10, x10, x14 ; Round 10 - mov x13, v5.d[0] + mov x13, V5.D[0] ldr x15, [x3], #8 ror x12, x6, #14 ror x14, x10, #28 @@ -455,47 +455,47 @@ L_sha512_len_neon_start add x5, x5, x9 add x9, x9, x14 ; Round 11 - mov x13, v5.d[1] + mov x13, V5.D[1] ldr x15, [x3], #8 - ext v10.16b, v5.16b, v6.16b, #8 + ext8 V10.16B, V5.16B, V6.16B, #8 ror x12, x5, #14 - shl v8.2d, v4.2d, #45 + shl V8.2D, V4.2D, #45 ror x14, x9, #28 - sri v8.2d, v4.2d, #19 + sri V8.2D, V4.2D, #19 eor x12, x12, x5, ror 18 - shl v9.2d, v4.2d, #3 + shl V9.2D, V4.2D, #3 eor x14, x14, x9, ror 34 - sri v9.2d, v4.2d, #61 + sri V9.2D, V4.2D, #61 eor x12, x12, x5, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x9, ror 39 - ushr v8.2d, v4.2d, #6 + ushr V8.2D, V4.2D, #6 add x8, x8, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x9, x10 - add v5.2d, v5.2d, v9.2d + add V5.2D, V5.2D, V9.2D eor x12, x6, x7 - ext v9.16b, v1.16b, v2.16b, #8 + ext8 V9.16B, V1.16B, V2.16B, #8 and x17, x16, x17 - add v5.2d, v5.2d, v9.2d + add V5.2D, V5.2D, V9.2D and x12, x12, x5 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x8, x8, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x7 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x8, x8, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x10 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x8, x8, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v5.2d, v5.2d, v9.2d + add V5.2D, V5.2D, V9.2D add x4, x4, x8 add x8, x8, x14 ; Round 12 - mov x13, v6.d[0] + mov x13, V6.D[0] ldr x15, [x3], #8 ror x12, x4, #14 ror x14, x8, #28 @@ -517,47 +517,47 @@ L_sha512_len_neon_start add x11, x11, x7 add x7, x7, x14 ; Round 13 - mov x13, v6.d[1] + mov x13, V6.D[1] ldr x15, [x3], #8 - ext v10.16b, v6.16b, v7.16b, #8 + ext8 V10.16B, V6.16B, V7.16B, #8 ror x12, x11, #14 - shl v8.2d, v5.2d, #45 + shl V8.2D, V5.2D, #45 ror x14, x7, #28 - sri v8.2d, v5.2d, #19 + sri V8.2D, V5.2D, #19 eor x12, x12, x11, ror 18 - shl v9.2d, v5.2d, #3 + shl V9.2D, V5.2D, #3 eor x14, x14, x7, ror 34 - sri v9.2d, v5.2d, #61 + sri V9.2D, V5.2D, #61 eor x12, x12, x11, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x7, ror 39 - ushr v8.2d, v5.2d, #6 + ushr V8.2D, V5.2D, #6 add x6, x6, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x7, x8 - add v6.2d, v6.2d, v9.2d + add V6.2D, V6.2D, V9.2D eor x12, x4, x5 - ext v9.16b, v2.16b, v3.16b, #8 + ext8 V9.16B, V2.16B, V3.16B, #8 and x17, x16, x17 - add v6.2d, v6.2d, v9.2d + add V6.2D, V6.2D, V9.2D and x12, x12, x11 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x6, x6, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x5 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x6, x6, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x8 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x6, x6, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v6.2d, v6.2d, v9.2d + add V6.2D, V6.2D, V9.2D add x10, x10, x6 add x6, x6, x14 ; Round 14 - mov x13, v7.d[0] + mov x13, V7.D[0] ldr x15, [x3], #8 ror x12, x10, #14 ror x14, x6, #28 @@ -579,49 +579,49 @@ L_sha512_len_neon_start add x9, x9, x5 add x5, x5, x14 ; Round 15 - mov x13, v7.d[1] + mov x13, V7.D[1] ldr x15, [x3], #8 - ext v10.16b, v7.16b, v0.16b, #8 + ext8 V10.16B, V7.16B, V0.16B, #8 ror x12, x9, #14 - shl v8.2d, v6.2d, #45 + shl V8.2D, V6.2D, #45 ror x14, x5, #28 - sri v8.2d, v6.2d, #19 + sri V8.2D, V6.2D, #19 eor x12, x12, x9, ror 18 - shl v9.2d, v6.2d, #3 + shl V9.2D, V6.2D, #3 eor x14, x14, x5, ror 34 - sri v9.2d, v6.2d, #61 + sri V9.2D, V6.2D, #61 eor x12, x12, x9, ror 41 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x14, x14, x5, ror 39 - ushr v8.2d, v6.2d, #6 + ushr V8.2D, V6.2D, #6 add x4, x4, x12 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x16, x5, x6 - add v7.2d, v7.2d, v9.2d + add V7.2D, V7.2D, V9.2D eor x12, x10, x11 - ext v9.16b, v3.16b, v4.16b, #8 + ext8 V9.16B, V3.16B, V4.16B, #8 and x17, x16, x17 - add v7.2d, v7.2d, v9.2d + add V7.2D, V7.2D, V9.2D and x12, x12, x9 - shl v8.2d, v10.2d, #63 + shl V8.2D, V10.2D, #63 add x4, x4, x13 - sri v8.2d, v10.2d, #1 + sri V8.2D, V10.2D, #1 eor x12, x12, x11 - tbl v9.16b, {v10.16b}, v11.16b + tbl V9.16B, {V10.16B}, V11.16B add x4, x4, x15 - eor v9.16b, v9.16b, v8.16b + eor V9.16B, V9.16B, V8.16B eor x17, x17, x6 - ushr v10.2d, v10.2d, #7 + ushr V10.2D, V10.2D, #7 add x4, x4, x12 - eor v9.16b, v9.16b, v10.16b + eor V9.16B, V9.16B, V10.16B add x14, x14, x17 - add v7.2d, v7.2d, v9.2d + add V7.2D, V7.2D, V9.2D add x8, x8, x4 add x4, x4, x14 subs x27, x27, #1 bne L_sha512_len_neon_start ; Round 0 - mov x13, v0.d[0] + mov x13, V0.D[0] ldr x15, [x3], #8 ror x12, x8, #14 ror x14, x4, #28 @@ -643,7 +643,7 @@ L_sha512_len_neon_start add x7, x7, x11 add x11, x11, x14 ; Round 1 - mov x13, v0.d[1] + mov x13, V0.D[1] ldr x15, [x3], #8 ror x12, x7, #14 ror x14, x11, #28 @@ -665,7 +665,7 @@ L_sha512_len_neon_start add x6, x6, x10 add x10, x10, x14 ; Round 2 - mov x13, v1.d[0] + mov x13, V1.D[0] ldr x15, [x3], #8 ror x12, x6, #14 ror x14, x10, #28 @@ -687,7 +687,7 @@ L_sha512_len_neon_start add x5, x5, x9 add x9, x9, x14 ; Round 3 - mov x13, v1.d[1] + mov x13, V1.D[1] ldr x15, [x3], #8 ror x12, x5, #14 ror x14, x9, #28 @@ -709,7 +709,7 @@ L_sha512_len_neon_start add x4, x4, x8 add x8, x8, x14 ; Round 4 - mov x13, v2.d[0] + mov x13, V2.D[0] ldr x15, [x3], #8 ror x12, x4, #14 ror x14, x8, #28 @@ -731,7 +731,7 @@ L_sha512_len_neon_start add x11, x11, x7 add x7, x7, x14 ; Round 5 - mov x13, v2.d[1] + mov x13, V2.D[1] ldr x15, [x3], #8 ror x12, x11, #14 ror x14, x7, #28 @@ -753,7 +753,7 @@ L_sha512_len_neon_start add x10, x10, x6 add x6, x6, x14 ; Round 6 - mov x13, v3.d[0] + mov x13, V3.D[0] ldr x15, [x3], #8 ror x12, x10, #14 ror x14, x6, #28 @@ -775,7 +775,7 @@ L_sha512_len_neon_start add x9, x9, x5 add x5, x5, x14 ; Round 7 - mov x13, v3.d[1] + mov x13, V3.D[1] ldr x15, [x3], #8 ror x12, x9, #14 ror x14, x5, #28 @@ -797,7 +797,7 @@ L_sha512_len_neon_start add x8, x8, x4 add x4, x4, x14 ; Round 8 - mov x13, v4.d[0] + mov x13, V4.D[0] ldr x15, [x3], #8 ror x12, x8, #14 ror x14, x4, #28 @@ -819,7 +819,7 @@ L_sha512_len_neon_start add x7, x7, x11 add x11, x11, x14 ; Round 9 - mov x13, v4.d[1] + mov x13, V4.D[1] ldr x15, [x3], #8 ror x12, x7, #14 ror x14, x11, #28 @@ -841,7 +841,7 @@ L_sha512_len_neon_start add x6, x6, x10 add x10, x10, x14 ; Round 10 - mov x13, v5.d[0] + mov x13, V5.D[0] ldr x15, [x3], #8 ror x12, x6, #14 ror x14, x10, #28 @@ -863,7 +863,7 @@ L_sha512_len_neon_start add x5, x5, x9 add x9, x9, x14 ; Round 11 - mov x13, v5.d[1] + mov x13, V5.D[1] ldr x15, [x3], #8 ror x12, x5, #14 ror x14, x9, #28 @@ -885,7 +885,7 @@ L_sha512_len_neon_start add x4, x4, x8 add x8, x8, x14 ; Round 12 - mov x13, v6.d[0] + mov x13, V6.D[0] ldr x15, [x3], #8 ror x12, x4, #14 ror x14, x8, #28 @@ -907,7 +907,7 @@ L_sha512_len_neon_start add x11, x11, x7 add x7, x7, x14 ; Round 13 - mov x13, v6.d[1] + mov x13, V6.D[1] ldr x15, [x3], #8 ror x12, x11, #14 ror x14, x7, #28 @@ -929,7 +929,7 @@ L_sha512_len_neon_start add x10, x10, x6 add x6, x6, x14 ; Round 14 - mov x13, v7.d[0] + mov x13, V7.D[0] ldr x15, [x3], #8 ror x12, x10, #14 ror x14, x6, #28 @@ -951,7 +951,7 @@ L_sha512_len_neon_start add x9, x9, x5 add x5, x5, x14 ; Round 15 - mov x13, v7.d[1] + mov x13, V7.D[1] ldr x15, [x3], #8 ror x12, x9, #14 ror x14, x5, #28 @@ -992,13 +992,13 @@ L_sha512_len_neon_start ldp x22, x23, [x29, #48] ldp x24, x25, [x29, #64] ldp x26, x27, [x29, #80] - ldp d8, d9, [x29, #96] - ldp d10, d11, [x29, #112] + ldp D8, D9, [x29, #96] + ldp D10, D11, [x29, #112] ldp x29, x30, [sp], #0x80 ret ENDP IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA512 - AREA |.rodata|, DATA, READONLY + AREA |.rodata|, DATA, READONLY, ALIGN=4 ALIGN 16 L_SHA512_trans_crypto_len_k DCQ 0x428a2f98d728ae22, 0x7137449123ef65cd @@ -1047,522 +1047,522 @@ L_SHA512_trans_crypto_len_k Transform_Sha512_Len_crypto PROC stp x29, x30, [sp, #-80]! add x29, sp, #0 - stp d8, d9, [x29, #16] - stp d10, d11, [x29, #32] - stp d12, d13, [x29, #48] - stp d14, d15, [x29, #64] + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] adrp x4, L_SHA512_trans_crypto_len_k add x4, x4, L_SHA512_trans_crypto_len_k ; .arch_extension sha3 ; Load K into vector registers - ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x4], #0x40 - ld1 {v12.2d, v13.2d, v14.2d, v15.2d}, [x4], #0x40 + ld1 {V8.2D, V9.2D, V10.2D, V11.2D}, [x4], #0x40 + ld1 {V12.2D, V13.2D, V14.2D, V15.2D}, [x4], #0x40 ; Load digest into working vars - ld1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x0] + ld1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x0] ; Start of loop processing a block L_sha512_len_crypto_begin mov x3, x4 ; Load W - ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #0x40 - ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #0x40 - rev64 v0.16b, v0.16b - rev64 v1.16b, v1.16b - rev64 v2.16b, v2.16b - rev64 v3.16b, v3.16b - rev64 v4.16b, v4.16b - rev64 v5.16b, v5.16b - rev64 v6.16b, v6.16b - rev64 v7.16b, v7.16b + ld1 {V0.16B, V1.16B, V2.16B, V3.16B}, [x1], #0x40 + ld1 {V4.16B, V5.16B, V6.16B, V7.16B}, [x1], #0x40 + rev64 V0.16B, V0.16B + rev64 V1.16B, V1.16B + rev64 V2.16B, V2.16B + rev64 V3.16B, V3.16B + rev64 V4.16B, V4.16B + rev64 V5.16B, V5.16B + rev64 V6.16B, V6.16B + rev64 V7.16B, V7.16B ; Copy digest to add in at end - mov v28.16b, v24.16b - mov v29.16b, v25.16b - mov v30.16b, v26.16b - mov v31.16b, v27.16b + mov V28.16B, V24.16B + mov V29.16B, V25.16B + mov V30.16B, V26.16B + mov V31.16B, V27.16B ; Start of 16 rounds ; Round 0 - add v20.2d, v0.2d, v8.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + add V20.2D, V0.2D, V8.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 1 - add v20.2d, v1.2d, v9.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + add V20.2D, V1.2D, V9.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 2 - add v20.2d, v2.2d, v10.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + add V20.2D, V2.2D, V10.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 3 - add v20.2d, v3.2d, v11.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + add V20.2D, V3.2D, V11.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 4 - add v20.2d, v4.2d, v12.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + add V20.2D, V4.2D, V12.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 5 - add v20.2d, v5.2d, v13.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + add V20.2D, V5.2D, V13.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 6 - add v20.2d, v6.2d, v14.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + add V20.2D, V6.2D, V14.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 7 - add v20.2d, v7.2d, v15.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + add V20.2D, V7.2D, V15.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 8 - sha512su0 v0.2d, v1.2d - ext v21.16b, v4.16b, v5.16b, #8 - sha512su1 v0.2d, v7.2d, v21.2d - add v20.2d, v0.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V0.2D, V1.2D + ext8 V21.16B, V4.16B, V5.16B, #8 + sha512su1 V0.2D, V7.2D, V21.2D + add V20.2D, V0.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 9 - sha512su0 v1.2d, v2.2d - ext v21.16b, v5.16b, v6.16b, #8 - sha512su1 v1.2d, v0.2d, v21.2d - add v20.2d, v1.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V1.2D, V2.2D + ext8 V21.16B, V5.16B, V6.16B, #8 + sha512su1 V1.2D, V0.2D, V21.2D + add V20.2D, V1.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 10 - sha512su0 v2.2d, v3.2d - ext v21.16b, v6.16b, v7.16b, #8 - sha512su1 v2.2d, v1.2d, v21.2d - add v20.2d, v2.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V2.2D, V3.2D + ext8 V21.16B, V6.16B, V7.16B, #8 + sha512su1 V2.2D, V1.2D, V21.2D + add V20.2D, V2.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 11 - sha512su0 v3.2d, v4.2d - ext v21.16b, v7.16b, v0.16b, #8 - sha512su1 v3.2d, v2.2d, v21.2d - add v20.2d, v3.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V3.2D, V4.2D + ext8 V21.16B, V7.16B, V0.16B, #8 + sha512su1 V3.2D, V2.2D, V21.2D + add V20.2D, V3.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 12 - sha512su0 v4.2d, v5.2d - ext v21.16b, v0.16b, v1.16b, #8 - sha512su1 v4.2d, v3.2d, v21.2d - add v20.2d, v4.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V4.2D, V5.2D + ext8 V21.16B, V0.16B, V1.16B, #8 + sha512su1 V4.2D, V3.2D, V21.2D + add V20.2D, V4.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 13 - sha512su0 v5.2d, v6.2d - ext v21.16b, v1.16b, v2.16b, #8 - sha512su1 v5.2d, v4.2d, v21.2d - add v20.2d, v5.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V5.2D, V6.2D + ext8 V21.16B, V1.16B, V2.16B, #8 + sha512su1 V5.2D, V4.2D, V21.2D + add V20.2D, V5.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 14 - sha512su0 v6.2d, v7.2d - ext v21.16b, v2.16b, v3.16b, #8 - sha512su1 v6.2d, v5.2d, v21.2d - add v20.2d, v6.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V6.2D, V7.2D + ext8 V21.16B, V2.16B, V3.16B, #8 + sha512su1 V6.2D, V5.2D, V21.2D + add V20.2D, V6.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 15 - sha512su0 v7.2d, v0.2d - ext v21.16b, v3.16b, v4.16b, #8 - sha512su1 v7.2d, v6.2d, v21.2d - add v20.2d, v7.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V7.2D, V0.2D + ext8 V21.16B, V3.16B, V4.16B, #8 + sha512su1 V7.2D, V6.2D, V21.2D + add V20.2D, V7.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 16 - sha512su0 v0.2d, v1.2d - ext v21.16b, v4.16b, v5.16b, #8 - sha512su1 v0.2d, v7.2d, v21.2d - add v20.2d, v0.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V0.2D, V1.2D + ext8 V21.16B, V4.16B, V5.16B, #8 + sha512su1 V0.2D, V7.2D, V21.2D + add V20.2D, V0.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 17 - sha512su0 v1.2d, v2.2d - ext v21.16b, v5.16b, v6.16b, #8 - sha512su1 v1.2d, v0.2d, v21.2d - add v20.2d, v1.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V1.2D, V2.2D + ext8 V21.16B, V5.16B, V6.16B, #8 + sha512su1 V1.2D, V0.2D, V21.2D + add V20.2D, V1.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 18 - sha512su0 v2.2d, v3.2d - ext v21.16b, v6.16b, v7.16b, #8 - sha512su1 v2.2d, v1.2d, v21.2d - add v20.2d, v2.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V2.2D, V3.2D + ext8 V21.16B, V6.16B, V7.16B, #8 + sha512su1 V2.2D, V1.2D, V21.2D + add V20.2D, V2.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 19 - sha512su0 v3.2d, v4.2d - ext v21.16b, v7.16b, v0.16b, #8 - sha512su1 v3.2d, v2.2d, v21.2d - add v20.2d, v3.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V3.2D, V4.2D + ext8 V21.16B, V7.16B, V0.16B, #8 + sha512su1 V3.2D, V2.2D, V21.2D + add V20.2D, V3.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 20 - sha512su0 v4.2d, v5.2d - ext v21.16b, v0.16b, v1.16b, #8 - sha512su1 v4.2d, v3.2d, v21.2d - add v20.2d, v4.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V4.2D, V5.2D + ext8 V21.16B, V0.16B, V1.16B, #8 + sha512su1 V4.2D, V3.2D, V21.2D + add V20.2D, V4.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 21 - sha512su0 v5.2d, v6.2d - ext v21.16b, v1.16b, v2.16b, #8 - sha512su1 v5.2d, v4.2d, v21.2d - add v20.2d, v5.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V5.2D, V6.2D + ext8 V21.16B, V1.16B, V2.16B, #8 + sha512su1 V5.2D, V4.2D, V21.2D + add V20.2D, V5.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 22 - sha512su0 v6.2d, v7.2d - ext v21.16b, v2.16b, v3.16b, #8 - sha512su1 v6.2d, v5.2d, v21.2d - add v20.2d, v6.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V6.2D, V7.2D + ext8 V21.16B, V2.16B, V3.16B, #8 + sha512su1 V6.2D, V5.2D, V21.2D + add V20.2D, V6.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 23 - sha512su0 v7.2d, v0.2d - ext v21.16b, v3.16b, v4.16b, #8 - sha512su1 v7.2d, v6.2d, v21.2d - add v20.2d, v7.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V7.2D, V0.2D + ext8 V21.16B, V3.16B, V4.16B, #8 + sha512su1 V7.2D, V6.2D, V21.2D + add V20.2D, V7.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 24 - sha512su0 v0.2d, v1.2d - ext v21.16b, v4.16b, v5.16b, #8 - sha512su1 v0.2d, v7.2d, v21.2d - add v20.2d, v0.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V0.2D, V1.2D + ext8 V21.16B, V4.16B, V5.16B, #8 + sha512su1 V0.2D, V7.2D, V21.2D + add V20.2D, V0.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 25 - sha512su0 v1.2d, v2.2d - ext v21.16b, v5.16b, v6.16b, #8 - sha512su1 v1.2d, v0.2d, v21.2d - add v20.2d, v1.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V1.2D, V2.2D + ext8 V21.16B, V5.16B, V6.16B, #8 + sha512su1 V1.2D, V0.2D, V21.2D + add V20.2D, V1.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 26 - sha512su0 v2.2d, v3.2d - ext v21.16b, v6.16b, v7.16b, #8 - sha512su1 v2.2d, v1.2d, v21.2d - add v20.2d, v2.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V2.2D, V3.2D + ext8 V21.16B, V6.16B, V7.16B, #8 + sha512su1 V2.2D, V1.2D, V21.2D + add V20.2D, V2.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 27 - sha512su0 v3.2d, v4.2d - ext v21.16b, v7.16b, v0.16b, #8 - sha512su1 v3.2d, v2.2d, v21.2d - add v20.2d, v3.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V3.2D, V4.2D + ext8 V21.16B, V7.16B, V0.16B, #8 + sha512su1 V3.2D, V2.2D, V21.2D + add V20.2D, V3.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 28 - sha512su0 v4.2d, v5.2d - ext v21.16b, v0.16b, v1.16b, #8 - sha512su1 v4.2d, v3.2d, v21.2d - add v20.2d, v4.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V4.2D, V5.2D + ext8 V21.16B, V0.16B, V1.16B, #8 + sha512su1 V4.2D, V3.2D, V21.2D + add V20.2D, V4.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 29 - sha512su0 v5.2d, v6.2d - ext v21.16b, v1.16b, v2.16b, #8 - sha512su1 v5.2d, v4.2d, v21.2d - add v20.2d, v5.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V5.2D, V6.2D + ext8 V21.16B, V1.16B, V2.16B, #8 + sha512su1 V5.2D, V4.2D, V21.2D + add V20.2D, V5.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 30 - sha512su0 v6.2d, v7.2d - ext v21.16b, v2.16b, v3.16b, #8 - sha512su1 v6.2d, v5.2d, v21.2d - add v20.2d, v6.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V6.2D, V7.2D + ext8 V21.16B, V2.16B, V3.16B, #8 + sha512su1 V6.2D, V5.2D, V21.2D + add V20.2D, V6.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Round 31 - sha512su0 v7.2d, v0.2d - ext v21.16b, v3.16b, v4.16b, #8 - sha512su1 v7.2d, v6.2d, v21.2d - add v20.2d, v7.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V7.2D, V0.2D + ext8 V21.16B, V3.16B, V4.16B, #8 + sha512su1 V7.2D, V6.2D, V21.2D + add V20.2D, V7.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 32 - sha512su0 v0.2d, v1.2d - ext v21.16b, v4.16b, v5.16b, #8 - sha512su1 v0.2d, v7.2d, v21.2d - add v20.2d, v0.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V0.2D, V1.2D + ext8 V21.16B, V4.16B, V5.16B, #8 + sha512su1 V0.2D, V7.2D, V21.2D + add V20.2D, V0.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 33 - sha512su0 v1.2d, v2.2d - ext v21.16b, v5.16b, v6.16b, #8 - sha512su1 v1.2d, v0.2d, v21.2d - add v20.2d, v1.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V1.2D, V2.2D + ext8 V21.16B, V5.16B, V6.16B, #8 + sha512su1 V1.2D, V0.2D, V21.2D + add V20.2D, V1.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 34 - sha512su0 v2.2d, v3.2d - ext v21.16b, v6.16b, v7.16b, #8 - sha512su1 v2.2d, v1.2d, v21.2d - add v20.2d, v2.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d + sha512su0 V2.2D, V3.2D + ext8 V21.16B, V6.16B, V7.16B, #8 + sha512su1 V2.2D, V1.2D, V21.2D + add V20.2D, V2.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D ; Round 35 - sha512su0 v3.2d, v4.2d - ext v21.16b, v7.16b, v0.16b, #8 - sha512su1 v3.2d, v2.2d, v21.2d - add v20.2d, v3.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v26.16b, v27.16b, #8 - ext v22.16b, v25.16b, v26.16b, #8 - add v27.2d, v27.2d, v20.2d - sha512h q27, q21, v22.2d - add v23.2d, v25.2d, v27.2d - sha512h2 q27, q25, v24.2d + sha512su0 V3.2D, V4.2D + ext8 V21.16B, V7.16B, V0.16B, #8 + sha512su1 V3.2D, V2.2D, V21.2D + add V20.2D, V3.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V26.16B, V27.16B, #8 + ext8 V22.16B, V25.16B, V26.16B, #8 + add V27.2D, V27.2D, V20.2D + sha512h Q27, Q21, V22.2D + add V23.2D, V25.2D, V27.2D + sha512h2 Q27, Q25, V24.2D ; Load next 8 64-bit words of K - ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x3], #0x40 + ld1 {V16.2D, V17.2D, V18.2D, V19.2D}, [x3], #0x40 ; Round 36 - sha512su0 v4.2d, v5.2d - ext v21.16b, v0.16b, v1.16b, #8 - sha512su1 v4.2d, v3.2d, v21.2d - add v20.2d, v4.2d, v16.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v23.16b, v26.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - add v26.2d, v26.2d, v20.2d - sha512h q26, q21, v22.2d - add v25.2d, v24.2d, v26.2d - sha512h2 q26, q24, v27.2d + sha512su0 V4.2D, V5.2D + ext8 V21.16B, V0.16B, V1.16B, #8 + sha512su1 V4.2D, V3.2D, V21.2D + add V20.2D, V4.2D, V16.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V23.16B, V26.16B, #8 + ext8 V22.16B, V24.16B, V23.16B, #8 + add V26.2D, V26.2D, V20.2D + sha512h Q26, Q21, V22.2D + add V25.2D, V24.2D, V26.2D + sha512h2 Q26, Q24, V27.2D ; Round 37 - sha512su0 v5.2d, v6.2d - ext v21.16b, v1.16b, v2.16b, #8 - sha512su1 v5.2d, v4.2d, v21.2d - add v20.2d, v5.2d, v17.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v25.16b, v23.16b, #8 - ext v22.16b, v27.16b, v25.16b, #8 - add v23.2d, v23.2d, v20.2d - sha512h q23, q21, v22.2d - add v24.2d, v27.2d, v23.2d - sha512h2 q23, q27, v26.2d + sha512su0 V5.2D, V6.2D + ext8 V21.16B, V1.16B, V2.16B, #8 + sha512su1 V5.2D, V4.2D, V21.2D + add V20.2D, V5.2D, V17.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V25.16B, V23.16B, #8 + ext8 V22.16B, V27.16B, V25.16B, #8 + add V23.2D, V23.2D, V20.2D + sha512h Q23, Q21, V22.2D + add V24.2D, V27.2D, V23.2D + sha512h2 Q23, Q27, V26.2D ; Round 38 - sha512su0 v6.2d, v7.2d - ext v21.16b, v2.16b, v3.16b, #8 - sha512su1 v6.2d, v5.2d, v21.2d - add v20.2d, v6.2d, v18.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v24.16b, v25.16b, #8 - ext v22.16b, v26.16b, v24.16b, #8 - add v25.2d, v25.2d, v20.2d - sha512h q25, q21, v22.2d - add v27.2d, v26.2d, v25.2d - sha512h2 q25, q26, v23.2d + sha512su0 V6.2D, V7.2D + ext8 V21.16B, V2.16B, V3.16B, #8 + sha512su1 V6.2D, V5.2D, V21.2D + add V20.2D, V6.2D, V18.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V24.16B, V25.16B, #8 + ext8 V22.16B, V26.16B, V24.16B, #8 + add V25.2D, V25.2D, V20.2D + sha512h Q25, Q21, V22.2D + add V27.2D, V26.2D, V25.2D + sha512h2 Q25, Q26, V23.2D ; Round 39 - sha512su0 v7.2d, v0.2d - ext v21.16b, v3.16b, v4.16b, #8 - sha512su1 v7.2d, v6.2d, v21.2d - add v20.2d, v7.2d, v19.2d - ext v20.16b, v20.16b, v20.16b, #8 - ext v21.16b, v27.16b, v24.16b, #8 - ext v22.16b, v23.16b, v27.16b, #8 - add v24.2d, v24.2d, v20.2d - sha512h q24, q21, v22.2d - add v26.2d, v23.2d, v24.2d - sha512h2 q24, q23, v25.2d - add v27.2d, v27.2d, v31.2d - add v26.2d, v26.2d, v30.2d - add v25.2d, v25.2d, v29.2d - add v24.2d, v24.2d, v28.2d + sha512su0 V7.2D, V0.2D + ext8 V21.16B, V3.16B, V4.16B, #8 + sha512su1 V7.2D, V6.2D, V21.2D + add V20.2D, V7.2D, V19.2D + ext8 V20.16B, V20.16B, V20.16B, #8 + ext8 V21.16B, V27.16B, V24.16B, #8 + ext8 V22.16B, V23.16B, V27.16B, #8 + add V24.2D, V24.2D, V20.2D + sha512h Q24, Q21, V22.2D + add V26.2D, V23.2D, V24.2D + sha512h2 Q24, Q23, V25.2D + add V27.2D, V27.2D, V31.2D + add V26.2D, V26.2D, V30.2D + add V25.2D, V25.2D, V29.2D + add V24.2D, V24.2D, V28.2D subs w2, w2, #0x80 bne L_sha512_len_crypto_begin ; Store digest back - st1 {v24.2d, v25.2d, v26.2d, v27.2d}, [x0] - ldp d8, d9, [x29, #16] - ldp d10, d11, [x29, #32] - ldp d12, d13, [x29, #48] - ldp d14, d15, [x29, #64] + st1 {V24.2D, V25.2D, V26.2D, V27.2D}, [x0] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] ldp x29, x30, [sp], #0x50 ret ENDP diff --git a/wolfcrypt/src/wc_mldsa_asm.asm b/wolfcrypt/src/wc_mldsa_asm.asm index 16ad868c23..2495d4894a 100644 --- a/wolfcrypt/src/wc_mldsa_asm.asm +++ b/wolfcrypt/src/wc_mldsa_asm.asm @@ -27314,6 +27314,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_0: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27398,6 +27400,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_1: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27482,6 +27486,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_2: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27566,6 +27572,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_3: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27650,6 +27658,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_4: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27734,6 +27744,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_5: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27818,6 +27830,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_6: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27902,6 +27916,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_7: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -27986,6 +28002,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_8: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28070,6 +28088,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_9: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28154,6 +28174,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_10: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28238,6 +28260,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_11: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28322,6 +28346,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_12: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28406,6 +28432,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_13: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28490,6 +28518,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_14: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28574,6 +28604,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_15: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28659,6 +28691,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_0: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28743,6 +28777,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_1: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28827,6 +28863,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_2: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28911,6 +28949,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_3: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -28995,6 +29035,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_4: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29079,6 +29121,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_5: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29163,6 +29207,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_6: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29247,6 +29293,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_7: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29331,6 +29379,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_8: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29415,6 +29465,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_9: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29499,6 +29551,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_10: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29583,6 +29637,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_11: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29667,6 +29723,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_12: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29751,6 +29809,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_13: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29835,6 +29895,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_14: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -29919,6 +29981,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_15: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30004,6 +30068,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_0: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30088,6 +30154,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_1: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30172,6 +30240,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_2: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30256,6 +30326,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_3: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30340,6 +30412,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_4: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30424,6 +30498,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_5: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30508,6 +30584,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_6: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30592,6 +30670,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_7: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30676,6 +30756,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_8: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30760,6 +30842,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_9: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30844,6 +30928,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_10: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -30928,6 +31014,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_11: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31012,6 +31100,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_12: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31096,6 +31186,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_13: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31180,6 +31272,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_14: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31264,6 +31358,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_15: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31349,6 +31445,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_0: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31433,6 +31531,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_1: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31517,6 +31617,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_2: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31601,6 +31703,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_3: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31685,6 +31789,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_4: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31769,6 +31875,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_5: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31853,6 +31961,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_6: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -31937,6 +32047,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_7: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32021,6 +32133,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_8: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32105,6 +32219,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_9: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32189,6 +32305,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_10: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32273,6 +32391,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_11: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32357,6 +32477,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_12: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32441,6 +32563,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_13: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32525,6 +32649,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_14: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32609,6 +32735,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_15: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32779,6 +32907,8 @@ L_mldsa_use_hint_32_avx2_hints_done__0: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32851,6 +32981,8 @@ L_mldsa_use_hint_32_avx2_hints_done__1: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32923,6 +33055,8 @@ L_mldsa_use_hint_32_avx2_hints_done__2: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -32995,6 +33129,8 @@ L_mldsa_use_hint_32_avx2_hints_done__3: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33067,6 +33203,8 @@ L_mldsa_use_hint_32_avx2_hints_done__4: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33139,6 +33277,8 @@ L_mldsa_use_hint_32_avx2_hints_done__5: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33211,6 +33351,8 @@ L_mldsa_use_hint_32_avx2_hints_done__6: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33283,6 +33425,8 @@ L_mldsa_use_hint_32_avx2_hints_done__7: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33355,6 +33499,8 @@ L_mldsa_use_hint_32_avx2_hints_done__8: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33427,6 +33573,8 @@ L_mldsa_use_hint_32_avx2_hints_done__9: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33499,6 +33647,8 @@ L_mldsa_use_hint_32_avx2_hints_done__10: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33571,6 +33721,8 @@ L_mldsa_use_hint_32_avx2_hints_done__11: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33643,6 +33795,8 @@ L_mldsa_use_hint_32_avx2_hints_done__12: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33715,6 +33869,8 @@ L_mldsa_use_hint_32_avx2_hints_done__13: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33787,6 +33943,8 @@ L_mldsa_use_hint_32_avx2_hints_done__14: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 @@ -33859,6 +34017,8 @@ L_mldsa_use_hint_32_avx2_hints_done__15: vpsllvd ymm7, ymm7, ymm13 vpsrad ymm6, ymm6, 31 vpsrad ymm7, ymm7, 31 + vpsubd ymm2, ymm2, ymm14 + vpsubd ymm3, ymm3, ymm14 vpsrld ymm2, ymm2, 31 vpsrld ymm3, ymm3, 31 vpslld ymm2, ymm2, 1 diff --git a/wolfssl-VS2022.vcxproj b/wolfssl-VS2022.vcxproj index 71cc6bbda2..672ead6059 100644 --- a/wolfssl-VS2022.vcxproj +++ b/wolfssl-VS2022.vcxproj @@ -76,7 +76,7 @@ the WOLFSSL_ARMASM* defines below); a symbol that is left undefined here simply drops the matching assembly (safe - the C caller then uses a C fallback), while a symbol enabled here but not in C is dead code. --> - --predefine "HAVE_AES_CBC SETL {TRUE}" --predefine "HAVE_AESCCM SETL {TRUE}" --predefine "HAVE_AESGCM SETL {TRUE}" --predefine "HAVE_AES_ECB SETL {TRUE}" --predefine "HAVE_AES_DECRYPT SETL {TRUE}" --predefine "WOLFSSL_AES_COUNTER SETL {TRUE}" --predefine "WOLFSSL_AES_DIRECT SETL {TRUE}" --predefine "WOLFSSL_AES_XTS SETL {TRUE}" --predefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" --predefine "WOLFSSL_SHA224 SETL {TRUE}" --predefine "WOLFSSL_SHA3 SETL {TRUE}" --predefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" --predefine "WOLFSSL_SHA384 SETL {TRUE}" --predefine "WOLFSSL_SHA512 SETL {TRUE}" --predefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" --predefine "HAVE_CHACHA SETL {TRUE}" --predefine "HAVE_CURVE25519 SETL {TRUE}" --predefine "HAVE_ED25519 SETL {TRUE}" --predefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" + -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" StaticLibrary diff --git a/wolfssl.vcxproj b/wolfssl.vcxproj index f565549501..c7ee00da15 100644 --- a/wolfssl.vcxproj +++ b/wolfssl.vcxproj @@ -75,7 +75,7 @@ the WOLFSSL_ARMASM* defines below); a symbol that is left undefined here simply drops the matching assembly (safe - the C caller then uses a C fallback), while a symbol enabled here but not in C is dead code. --> - --predefine "HAVE_AES_CBC SETL {TRUE}" --predefine "HAVE_AESCCM SETL {TRUE}" --predefine "HAVE_AESGCM SETL {TRUE}" --predefine "HAVE_AES_ECB SETL {TRUE}" --predefine "HAVE_AES_DECRYPT SETL {TRUE}" --predefine "WOLFSSL_AES_COUNTER SETL {TRUE}" --predefine "WOLFSSL_AES_DIRECT SETL {TRUE}" --predefine "WOLFSSL_AES_XTS SETL {TRUE}" --predefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" --predefine "WOLFSSL_SHA224 SETL {TRUE}" --predefine "WOLFSSL_SHA3 SETL {TRUE}" --predefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" --predefine "WOLFSSL_SHA384 SETL {TRUE}" --predefine "WOLFSSL_SHA512 SETL {TRUE}" --predefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" --predefine "HAVE_CHACHA SETL {TRUE}" --predefine "HAVE_CURVE25519 SETL {TRUE}" --predefine "HAVE_ED25519 SETL {TRUE}" --predefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" + -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" StaticLibrary diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index 07f5829f1e..3e1a4d3f12 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -1897,6 +1897,10 @@ WOLFSSL_ABI WOLFSSL_API int wolfCrypt_Cleanup(void); #define XFENCE() XASM_VOLATILE("lfence") #elif defined (__arm__) && (__ARM_ARCH > 6) #define XFENCE() XASM_VOLATILE("isb") + #elif defined(_MSC_VER) && defined(_M_ARM64) + /* MSVC on ARM64 has no __asm__; use the ISB intrinsic barrier. */ + #include + #define XFENCE() __isb(_ARM64_BARRIER_SY) #elif defined(__aarch64__) /* Change ".inst 0xd50330ff" to "sb" when compilers support it. */ #ifdef WOLFSSL_ARMASM_BARRIER_SB