From 649054f7b8c64363d86c7c6e53da4cccf0e2b98a Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Mon, 6 Jul 2026 20:04:54 +1000 Subject: [PATCH] Wycheproof: fixes for various platforms Fixes for C, Intel x64, Aarch64, ARM32, Thumb2, Thumb. --- wolfcrypt/src/ecc.c | 6 +- wolfcrypt/src/fe_448.c | 1 + wolfcrypt/src/fe_x25519_asm.S | 18 + wolfcrypt/src/port/arm/armv8-32-curve25519.S | 1091 +++++++++- .../src/port/arm/armv8-32-curve25519_c.c | 1175 ++++++++++- wolfcrypt/src/port/arm/armv8-aes-asm.S | 84 +- wolfcrypt/src/port/arm/armv8-aes-asm_c.c | 1811 ++++++++--------- wolfcrypt/src/port/arm/armv8-curve25519.S | 8 + wolfcrypt/src/port/arm/armv8-curve25519_c.c | 18 +- wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c | 12 +- wolfcrypt/src/port/arm/thumb2-curve25519.S | 1049 +++++++++- wolfcrypt/src/port/arm/thumb2-curve25519_c.c | 1125 +++++++++- wolfcrypt/src/sp_arm32.c | 45 +- wolfcrypt/src/sp_armthumb.c | 135 +- wolfcrypt/src/sp_c32.c | 6 + wolfcrypt/src/sp_c64.c | 4 + wolfcrypt/src/sp_cortexm.c | 45 +- wolfcrypt/src/sp_int.c | 61 +- wolfcrypt/src/wc_mldsa.c | 20 +- wolfcrypt/src/wc_mldsa_asm.S | 160 ++ 20 files changed, 5650 insertions(+), 1224 deletions(-) diff --git a/wolfcrypt/src/ecc.c b/wolfcrypt/src/ecc.c index ab8b42fb97..686836c38c 100644 --- a/wolfcrypt/src/ecc.c +++ b/wolfcrypt/src/ecc.c @@ -9345,9 +9345,11 @@ static int ecc_verify_hash(mp_int *r, mp_int *s, const byte* hash, if (err == MP_OKAY) err = mp_montgomery_setup(curve->prime, &mp); - /* add them */ + /* add them - pass mG as A (first) not B (second): the safe-add + * recovery for a missed-double doubles B into R, which requires + * B and R to be different points. */ if (err == MP_OKAY) - err = ecc_projective_add_point_safe(mQ, mG, mG, curve->Af, + err = ecc_projective_add_point_safe(mG, mQ, mG, curve->Af, curve->prime, mp, NULL); #ifdef WOLFSSL_CHECK_VER_FAULTS if (err == MP_OKAY && wc_ecc_cmp_point(mG, &mG1) == MP_EQ) { diff --git a/wolfcrypt/src/fe_448.c b/wolfcrypt/src/fe_448.c index 6288f6c8d6..a572822d58 100644 --- a/wolfcrypt/src/fe_448.c +++ b/wolfcrypt/src/fe_448.c @@ -2302,6 +2302,7 @@ int curve448(byte* r, const byte* n, const byte* a) fe448_reduce(x3); fe448_sqr(x3, x3); fe448_sub(z3, z3, t1); + fe448_reduce(z3); fe448_sqr(z3, z3); fe448_mul(z3, z3, x1); fe448_sub(t1, t0, x2); diff --git a/wolfcrypt/src/fe_x25519_asm.S b/wolfcrypt/src/fe_x25519_asm.S index 7e976fa1f2..69f1f61838 100644 --- a/wolfcrypt/src/fe_x25519_asm.S +++ b/wolfcrypt/src/fe_x25519_asm.S @@ -5738,6 +5738,15 @@ L_curve25519_x64_bits: adcq %r12, %r9 adcq %r13, %r10 adcq %r14, %r11 + movq $0x7fffffffffffffff, %rbp + movq %r11, %rax + sarq $63, %rax + andq $19, %rax + andq %rbp, %r11 + addq %rax, %rcx + adcq $0x00, %r9 + adcq $0x00, %r10 + adcq $0x00, %r11 # Store movq %rcx, 64(%rsp) movq %r9, 72(%rsp) @@ -16022,6 +16031,15 @@ L_curve25519_avx2_bits: adcxq %rbx, %r11 adoxq %r15, %r12 adcxq %rcx, %r12 + movq $0x7fffffffffffffff, %rcx + movq %r12, %rdx + sarq $63, %rdx + andq $19, %rdx + andq %rcx, %r12 + addq %rdx, %r9 + adcq $0x00, %r10 + adcq $0x00, %r11 + adcq $0x00, %r12 # Store movq %r9, 64(%rsp) movq %r10, 72(%rsp) diff --git a/wolfcrypt/src/port/arm/armv8-32-curve25519.S b/wolfcrypt/src/port/arm/armv8-32-curve25519.S index 539f3ac219..f03c1920cf 100644 --- a/wolfcrypt/src/port/arm/armv8-32-curve25519.S +++ b/wolfcrypt/src/port/arm/armv8-32-curve25519.S @@ -2607,6 +2607,410 @@ fe_mul_op: add sp, sp, #40 pop {pc} .size fe_mul_op,.-fe_mul_op + .text + .align 4 + .globl fe_mul_op_full_red + .type fe_mul_op_full_red, %function +fe_mul_op_full_red: + push {lr} + sub sp, sp, #40 + str r0, [sp, #36] + mov r0, #0 + ldr r12, [r1] + # A[0] * B[0] + ldr lr, [r2] + umull r3, r4, r12, lr + # A[0] * B[2] + ldr lr, [r2, #8] + umull r5, r6, r12, lr + # A[0] * B[4] + ldr lr, [r2, #16] + umull r7, r8, r12, lr + # A[0] * B[6] + ldr lr, [r2, #24] + umull r9, r10, r12, lr + str r3, [sp] + # A[0] * B[1] + ldr lr, [r2, #4] + mov r11, r0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[0] * B[3] + ldr lr, [r2, #12] + adcs r6, r6, #0 + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[0] * B[5] + ldr lr, [r2, #20] + adcs r8, r8, #0 + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[0] * B[7] + ldr lr, [r2, #28] + adcs r10, r10, #0 + adc r3, r0, #0 + umlal r10, r3, r12, lr + # A[1] * B[0] + ldr r12, [r1, #4] + ldr lr, [r2] + mov r11, #0 + umlal r4, r11, r12, lr + str r4, [sp, #4] + adds r5, r5, r11 + # A[1] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[1] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[1] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[1] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[1] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[1] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[1] * B[7] + ldr lr, [r2, #28] + adc r4, r0, #0 + umlal r3, r4, r12, lr + # A[2] * B[0] + ldr r12, [r1, #8] + ldr lr, [r2] + mov r11, #0 + umlal r5, r11, r12, lr + str r5, [sp, #8] + adds r6, r6, r11 + # A[2] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[2] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[2] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[2] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[2] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[2] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[2] * B[7] + ldr lr, [r2, #28] + adc r5, r0, #0 + umlal r4, r5, r12, lr + # A[3] * B[0] + ldr r12, [r1, #12] + ldr lr, [r2] + mov r11, #0 + umlal r6, r11, r12, lr + str r6, [sp, #12] + adds r7, r7, r11 + # A[3] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[3] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[3] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[3] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[3] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[3] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[3] * B[7] + ldr lr, [r2, #28] + adc r6, r0, #0 + umlal r5, r6, r12, lr + # A[4] * B[0] + ldr r12, [r1, #16] + ldr lr, [r2] + mov r11, #0 + umlal r7, r11, r12, lr + str r7, [sp, #16] + adds r8, r8, r11 + # A[4] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[4] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[4] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[4] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[4] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[4] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[4] * B[7] + ldr lr, [r2, #28] + adc r7, r0, #0 + umlal r6, r7, r12, lr + # A[5] * B[0] + ldr r12, [r1, #20] + ldr lr, [r2] + mov r11, #0 + umlal r8, r11, r12, lr + str r8, [sp, #20] + adds r9, r9, r11 + # A[5] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[5] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[5] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[5] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[5] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[5] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[5] * B[7] + ldr lr, [r2, #28] + adc r8, r0, #0 + umlal r7, r8, r12, lr + # A[6] * B[0] + ldr r12, [r1, #24] + ldr lr, [r2] + mov r11, #0 + umlal r9, r11, r12, lr + str r9, [sp, #24] + adds r10, r10, r11 + # A[6] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[6] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[6] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[6] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[6] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[6] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[6] * B[7] + ldr lr, [r2, #28] + adc r9, r0, #0 + umlal r8, r9, r12, lr + # A[7] * B[0] + ldr r12, [r1, #28] + ldr lr, [r2] + mov r11, #0 + umlal r10, r11, r12, lr + str r10, [sp, #28] + adds r3, r3, r11 + # A[7] * B[1] + ldr lr, [r2, #4] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[7] * B[2] + ldr lr, [r2, #8] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[7] * B[3] + ldr lr, [r2, #12] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[7] * B[4] + ldr lr, [r2, #16] + adc r11, r0, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[7] * B[5] + ldr lr, [r2, #20] + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[7] * B[6] + ldr lr, [r2, #24] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[7] * B[7] + ldr lr, [r2, #28] + adc r10, r0, #0 + umlal r9, r10, r12, lr + # Reduce + ldr r2, [sp, #28] + mov lr, sp + mov r12, #38 + umull r10, r11, r12, r10 + adds r10, r10, r2 + adc r11, r11, #0 + mov r12, #19 + lsl r11, r11, #1 + orr r11, r11, r10, lsr #31 + mul r11, r12, r11 + ldm lr!, {r1, r2} + mov r12, #38 + adds r1, r1, r11 + adc r11, r0, #0 + umlal r1, r11, r3, r12 + adds r2, r2, r11 + adc r11, r0, #0 + umlal r2, r11, r4, r12 + ldm lr!, {r3, r4} + adds r3, r3, r11 + adc r11, r0, #0 + umlal r3, r11, r5, r12 + adds r4, r4, r11 + adc r11, r0, #0 + umlal r4, r11, r6, r12 + ldm lr!, {r5, r6} + adds r5, r5, r11 + adc r11, r0, #0 + umlal r5, r11, r7, r12 + adds r6, r6, r11 + adc r11, r0, #0 + umlal r6, r11, r8, r12 + ldm lr!, {r7, r8} + adds r7, r7, r11 + adc r11, r0, #0 + umlal r7, r11, r9, r12 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r10, r10, #0x80000000 +#else + bfc r10, #31, #1 +#endif + adds r8, r10, r11 + adds r11, r1, #19 + adcs r11, r2, #0 + adcs r11, r3, #0 + adcs r11, r4, #0 + adcs r11, r5, #0 + adcs r11, r6, #0 + adcs r11, r7, #0 + adc r11, r8, #0 + asr r11, r11, #31 + and r11, r11, #19 + adds r1, r1, r11 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adcs r7, r7, #0 + adc r8, r8, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r8, r8, #0x80000000 +#else + bfc r8, #31, #1 +#endif + # Store + ldr r0, [sp, #36] + stm r0, {r1, r2, r3, r4, r5, r6, r7, r8} + add sp, sp, #40 + pop {pc} + .size fe_mul_op_full_red,.-fe_mul_op_full_red #else .text .align 4 @@ -2748,6 +3152,169 @@ fe_mul_op: add sp, sp, #16 pop {pc} .size fe_mul_op,.-fe_mul_op + .text + .align 4 + .globl fe_mul_op_full_red + .type fe_mul_op_full_red, %function +fe_mul_op_full_red: + push {lr} + sub sp, sp, #44 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + str r0, [sp, #36] + str r1, [sp, #40] +#else + strd r0, r1, [sp, #36] +#endif + mov lr, r2 + ldm r1, {r0, r1, r2, r3} + ldm lr!, {r4, r5, r6} + umull r10, r11, r0, r4 + umull r12, r7, r1, r4 + umaal r11, r12, r0, r5 + umull r8, r9, r2, r4 + umaal r12, r8, r1, r5 + umaal r12, r7, r0, r6 + umaal r8, r9, r3, r4 + stm sp, {r10, r11, r12} + umaal r7, r8, r2, r5 + ldm lr!, {r4} + umull r10, r11, r1, r6 + umaal r8, r9, r2, r6 + umaal r7, r10, r0, r4 + umaal r8, r11, r3, r5 + str r7, [sp, #12] + umaal r8, r10, r1, r4 + umaal r9, r11, r3, r6 + umaal r9, r10, r2, r4 + umaal r10, r11, r3, r4 + ldm lr, {r4, r5, r6, r7} + mov r12, #0 + umlal r8, r12, r0, r4 + umaal r9, r12, r1, r4 + umaal r10, r12, r2, r4 + umaal r11, r12, r3, r4 + mov r4, #0 + umlal r9, r4, r0, r5 + umaal r10, r4, r1, r5 + umaal r11, r4, r2, r5 + umaal r12, r4, r3, r5 + mov r5, #0 + umlal r10, r5, r0, r6 + umaal r11, r5, r1, r6 + umaal r12, r5, r2, r6 + umaal r4, r5, r3, r6 + mov r6, #0 + umlal r11, r6, r0, r7 + ldr r0, [sp, #40] + umaal r12, r6, r1, r7 + add r0, r0, #16 + umaal r4, r6, r2, r7 + sub lr, lr, #16 + umaal r5, r6, r3, r7 + ldm r0, {r0, r1, r2, r3} + str r6, [sp, #32] + ldm lr!, {r6} + mov r7, #0 + umlal r8, r7, r0, r6 + umaal r9, r7, r1, r6 + str r8, [sp, #16] + umaal r10, r7, r2, r6 + umaal r11, r7, r3, r6 + ldm lr!, {r6} + mov r8, #0 + umlal r9, r8, r0, r6 + umaal r10, r8, r1, r6 + str r9, [sp, #20] + umaal r11, r8, r2, r6 + umaal r12, r8, r3, r6 + ldm lr!, {r6} + mov r9, #0 + umlal r10, r9, r0, r6 + umaal r11, r9, r1, r6 + str r10, [sp, #24] + umaal r12, r9, r2, r6 + umaal r4, r9, r3, r6 + ldm lr!, {r6} + mov r10, #0 + umlal r11, r10, r0, r6 + umaal r12, r10, r1, r6 + str r11, [sp, #28] + umaal r4, r10, r2, r6 + umaal r5, r10, r3, r6 + ldm lr!, {r11} + umaal r12, r7, r0, r11 + umaal r4, r7, r1, r11 + ldr r6, [sp, #32] + umaal r5, r7, r2, r11 + umaal r6, r7, r3, r11 + ldm lr!, {r11} + umaal r4, r8, r0, r11 + umaal r5, r8, r1, r11 + umaal r6, r8, r2, r11 + umaal r7, r8, r3, r11 + ldm lr, {r11, lr} + umaal r5, r9, r0, r11 + umaal r6, r10, r0, lr + umaal r6, r9, r1, r11 + umaal r7, r10, r1, lr + umaal r7, r9, r2, r11 + umaal r8, r10, r2, lr + umaal r8, r9, r3, r11 + umaal r9, r10, r3, lr + # Reduce + ldr r0, [sp, #28] + mov lr, #37 + umaal r10, r0, r10, lr + mov lr, #19 + lsl r0, r0, #1 + orr r0, r0, r10, lsr #31 + mul r11, r0, lr + pop {r0-r2} + mov lr, #38 + umaal r0, r11, r12, lr + umaal r1, r11, r4, lr + umaal r2, r11, r5, lr + pop {r3-r5} + umaal r3, r11, r6, lr + umaal r4, r11, r7, lr + umaal r5, r11, r8, lr + pop {r6} +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r10, r10, #0x80000000 +#else + bfc r10, #31, #1 +#endif + umaal r6, r11, r9, lr + add r7, r10, r11 + adds r11, r0, #19 + adcs r11, r1, #0 + adcs r11, r2, #0 + adcs r11, r3, #0 + adcs r11, r4, #0 + adcs r11, r5, #0 + adcs r11, r6, #0 + adc r11, r7, #0 + asr r11, r11, #31 + and r11, r11, #19 + adds r0, r0, r11 + adcs r1, r1, #0 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adc r7, r7, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r7, r7, #0x80000000 +#else + bfc r7, #31, #1 +#endif + ldr lr, [sp, #8] + # Store + stm lr, {r0, r1, r2, r3, r4, r5, r6, r7} + add sp, sp, #16 + pop {pc} + .size fe_mul_op_full_red,.-fe_mul_op_full_red #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ .text .align 4 @@ -3033,6 +3600,303 @@ fe_sq_op: add sp, sp, #0x44 pop {pc} .size fe_sq_op,.-fe_sq_op + .text + .align 4 + .globl fe_sq_op_full_red + .type fe_sq_op_full_red, %function +fe_sq_op_full_red: + push {lr} + sub sp, sp, #0x44 + str r0, [sp, #64] + # Square + mov r0, #0 + ldr r12, [r1] + # A[0] * A[1] + ldr lr, [r1, #4] + umull r4, r5, r12, lr + # A[0] * A[3] + ldr lr, [r1, #12] + umull r6, r7, r12, lr + # A[0] * A[5] + ldr lr, [r1, #20] + umull r8, r9, r12, lr + # A[0] * A[7] + ldr lr, [r1, #28] + umull r10, r3, r12, lr + # A[0] * A[2] + ldr lr, [r1, #8] + mov r11, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[0] * A[4] + ldr lr, [r1, #16] + adcs r7, r7, #0 + adc r11, r0, #0 + umlal r7, r11, r12, lr + adds r8, r8, r11 + # A[0] * A[6] + ldr lr, [r1, #24] + adcs r9, r9, #0 + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + adcs r3, r3, #0 + str r4, [sp, #4] + str r5, [sp, #8] + # A[1] * A[2] + ldr r12, [r1, #4] + ldr lr, [r1, #8] + mov r11, #0 + umlal r6, r11, r12, lr + str r6, [sp, #12] + adds r7, r7, r11 + # A[1] * A[3] + ldr lr, [r1, #12] + adc r11, r0, #0 + umlal r7, r11, r12, lr + str r7, [sp, #16] + adds r8, r8, r11 + # A[1] * A[4] + ldr lr, [r1, #16] + adc r11, r0, #0 + umlal r8, r11, r12, lr + adds r9, r9, r11 + # A[1] * A[5] + ldr lr, [r1, #20] + adc r11, r0, #0 + umlal r9, r11, r12, lr + adds r10, r10, r11 + # A[1] * A[6] + ldr lr, [r1, #24] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[1] * A[7] + ldr lr, [r1, #28] + adc r4, r0, #0 + umlal r3, r4, r12, lr + # A[2] * A[3] + ldr r12, [r1, #8] + ldr lr, [r1, #12] + mov r11, #0 + umlal r8, r11, r12, lr + str r8, [sp, #20] + adds r9, r9, r11 + # A[2] * A[4] + ldr lr, [r1, #16] + adc r11, r0, #0 + umlal r9, r11, r12, lr + str r9, [sp, #24] + adds r10, r10, r11 + # A[2] * A[5] + ldr lr, [r1, #20] + adc r11, r0, #0 + umlal r10, r11, r12, lr + adds r3, r3, r11 + # A[2] * A[6] + ldr lr, [r1, #24] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[2] * A[7] + ldr lr, [r1, #28] + adc r5, r0, #0 + umlal r4, r5, r12, lr + # A[3] * A[4] + ldr r12, [r1, #12] + ldr lr, [r1, #16] + mov r11, #0 + umlal r10, r11, r12, lr + str r10, [sp, #28] + adds r3, r3, r11 + # A[3] * A[5] + ldr lr, [r1, #20] + adc r11, r0, #0 + umlal r3, r11, r12, lr + adds r4, r4, r11 + # A[3] * A[6] + ldr lr, [r1, #24] + adc r11, r0, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[3] * A[7] + ldr lr, [r1, #28] + adc r6, r0, #0 + umlal r5, r6, r12, lr + # A[4] * A[5] + ldr r12, [r1, #16] + ldr lr, [r1, #20] + mov r11, #0 + umlal r4, r11, r12, lr + adds r5, r5, r11 + # A[4] * A[6] + ldr lr, [r1, #24] + adc r11, r0, #0 + umlal r5, r11, r12, lr + adds r6, r6, r11 + # A[4] * A[7] + ldr lr, [r1, #28] + adc r7, r0, #0 + umlal r6, r7, r12, lr + # A[5] * A[6] + ldr r12, [r1, #20] + ldr lr, [r1, #24] + mov r11, #0 + umlal r6, r11, r12, lr + adds r7, r7, r11 + # A[5] * A[7] + ldr lr, [r1, #28] + adc r8, r0, #0 + umlal r7, r8, r12, lr + # A[6] * A[7] + ldr r12, [r1, #24] + ldr lr, [r1, #28] + mov r9, #0 + umlal r8, r9, r12, lr + add lr, sp, #32 + stm lr, {r3, r4, r5, r6, r7, r8, r9} + add lr, sp, #4 + ldm lr, {r4, r5, r6, r7, r8, r9, r10} + adds r4, r4, r4 + adcs r5, r5, r5 + adcs r6, r6, r6 + adcs r7, r7, r7 + adcs r8, r8, r8 + adcs r9, r9, r9 + adcs r10, r10, r10 + stm lr!, {r4, r5, r6, r7, r8, r9, r10} + ldm lr, {r3, r4, r5, r6, r7, r8, r9} + adcs r3, r3, r3 + adcs r4, r4, r4 + adcs r5, r5, r5 + adcs r6, r6, r6 + adcs r7, r7, r7 + adcs r8, r8, r8 + adcs r9, r9, r9 + adc r10, r0, #0 + stm lr, {r3, r4, r5, r6, r7, r8, r9, r10} + add lr, sp, #4 + ldm lr, {r4, r5, r6, r7, r8, r9, r10} + mov lr, sp + # A[0] * A[0] + ldr r12, [r1] + umull r3, r11, r12, r12 + adds r4, r4, r11 + # A[1] * A[1] + ldr r12, [r1, #4] + adcs r5, r5, #0 + adc r11, r0, #0 + umlal r5, r11, r12, r12 + adds r6, r6, r11 + # A[2] * A[2] + ldr r12, [r1, #8] + adcs r7, r7, #0 + adc r11, r0, #0 + umlal r7, r11, r12, r12 + adds r8, r8, r11 + # A[3] * A[3] + ldr r12, [r1, #12] + adcs r9, r9, #0 + adc r11, r0, #0 + umlal r9, r11, r12, r12 + adds r10, r10, r11 + stm lr!, {r3, r4, r5, r6, r7, r8, r9, r10} + ldm lr, {r3, r4, r5, r6, r7, r8, r9, r10} + # A[4] * A[4] + ldr r12, [r1, #16] + adcs r3, r3, #0 + adc r11, r0, #0 + umlal r3, r11, r12, r12 + adds r4, r4, r11 + # A[5] * A[5] + ldr r12, [r1, #20] + adcs r5, r5, #0 + adc r11, r0, #0 + umlal r5, r11, r12, r12 + adds r6, r6, r11 + # A[6] * A[6] + ldr r12, [r1, #24] + adcs r7, r7, #0 + adc r11, r0, #0 + umlal r7, r11, r12, r12 + adds r8, r8, r11 + # A[7] * A[7] + ldr r12, [r1, #28] + adcs r9, r9, #0 + adc r10, r10, #0 + umlal r9, r10, r12, r12 + # Reduce + ldr r2, [sp, #28] + mov lr, sp + mov r12, #38 + umull r10, r11, r12, r10 + adds r10, r10, r2 + adc r11, r11, #0 + mov r12, #19 + lsl r11, r11, #1 + orr r11, r11, r10, lsr #31 + mul r11, r12, r11 + ldm lr!, {r1, r2} + mov r12, #38 + adds r1, r1, r11 + adc r11, r0, #0 + umlal r1, r11, r3, r12 + adds r2, r2, r11 + adc r11, r0, #0 + umlal r2, r11, r4, r12 + ldm lr!, {r3, r4} + adds r3, r3, r11 + adc r11, r0, #0 + umlal r3, r11, r5, r12 + adds r4, r4, r11 + adc r11, r0, #0 + umlal r4, r11, r6, r12 + ldm lr!, {r5, r6} + adds r5, r5, r11 + adc r11, r0, #0 + umlal r5, r11, r7, r12 + adds r6, r6, r11 + adc r11, r0, #0 + umlal r6, r11, r8, r12 + ldm lr!, {r7, r8} + adds r7, r7, r11 + adc r11, r0, #0 + umlal r7, r11, r9, r12 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r10, r10, #0x80000000 +#else + bfc r10, #31, #1 +#endif + adds r8, r10, r11 + adds r11, r1, #19 + adcs r11, r2, #0 + adcs r11, r3, #0 + adcs r11, r4, #0 + adcs r11, r5, #0 + adcs r11, r6, #0 + adcs r11, r7, #0 + adc r11, r8, #0 + asr r11, r11, #31 + and r11, r11, #19 + adds r1, r1, r11 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adcs r7, r7, #0 + adc r8, r8, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r8, r8, #0x80000000 +#else + bfc r8, #31, #1 +#endif + # Store + ldr r0, [sp, #64] + stm r0, {r1, r2, r3, r4, r5, r6, r7, r8} + add sp, sp, #0x44 + pop {pc} + .size fe_sq_op_full_red,.-fe_sq_op_full_red #else .text .align 4 @@ -3160,6 +4024,155 @@ fe_sq_op: stm lr, {r0, r1, r2, r3, r4, r5, r6, r7} pop {pc} .size fe_sq_op,.-fe_sq_op + .text + .align 4 + .globl fe_sq_op_full_red + .type fe_sq_op_full_red, %function +fe_sq_op_full_red: + push {lr} + sub sp, sp, #32 + str r0, [sp, #28] + ldm r1, {r0, r1, r2, r3, r4, r5, r6, r7} + # Square + umull r9, r10, r0, r0 + umull r11, r12, r0, r1 + adds r11, r11, r11 + mov lr, #0 + umaal r10, r11, lr, lr + stm sp, {r9, r10} + mov r8, lr + umaal r8, r12, r0, r2 + adcs r8, r8, r8 + umaal r8, r11, r1, r1 + umull r9, r10, r0, r3 + umaal r9, r12, r1, r2 + adcs r9, r9, r9 + umaal r9, r11, lr, lr +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + str r8, [sp, #8] + str r9, [sp, #12] +#else + strd r8, r9, [sp, #8] +#endif + mov r9, lr + umaal r9, r10, r0, r4 + umaal r9, r12, r1, r3 + adcs r9, r9, r9 + umaal r9, r11, r2, r2 + str r9, [sp, #16] + umull r9, r8, r0, r5 + umaal r9, r12, r1, r4 + umaal r9, r10, r2, r3 + adcs r9, r9, r9 + umaal r9, r11, lr, lr + str r9, [sp, #20] + mov r9, lr + umaal r9, r8, r0, r6 + umaal r9, r12, r1, r5 + umaal r9, r10, r2, r4 + adcs r9, r9, r9 + umaal r9, r11, r3, r3 + str r9, [sp, #24] + umull r0, r9, r0, r7 + umaal r0, r8, r1, r6 + umaal r0, r12, r2, r5 + umaal r0, r10, r3, r4 + adcs r0, r0, r0 + umaal r0, r11, lr, lr + # R[7] = r0 + umaal r9, r8, r1, r7 + umaal r9, r10, r2, r6 + umaal r12, r9, r3, r5 + adcs r12, r12, r12 + umaal r12, r11, r4, r4 + # R[8] = r12 + umaal r9, r8, r2, r7 + umaal r10, r9, r3, r6 + mov r2, lr + umaal r10, r2, r4, r5 + adcs r10, r10, r10 + umaal r11, r10, lr, lr + # R[9] = r11 + umaal r2, r8, r3, r7 + umaal r2, r9, r4, r6 + adcs r3, r2, r2 + umaal r10, r3, r5, r5 + # R[10] = r10 + mov r1, lr + umaal r1, r8, r4, r7 + umaal r1, r9, r5, r6 + adcs r4, r1, r1 + umaal r3, r4, lr, lr + # R[11] = r3 + umaal r8, r9, r5, r7 + adcs r8, r8, r8 + umaal r4, r8, r6, r6 + # R[12] = r4 + mov r5, lr + umaal r5, r9, r6, r7 + adcs r5, r5, r5 + umaal r8, r5, lr, lr + # R[13] = r8 + adcs r9, r9, r9 + umaal r9, r5, r7, r7 + adcs r7, r5, lr + # R[14] = r9 + # R[15] = r7 + # Reduce + mov r6, #37 + umaal r7, r0, r7, r6 + mov r6, #19 + lsl r0, r0, #1 + orr r0, r0, r7, lsr #31 + mul lr, r0, r6 + pop {r0-r1} + mov r6, #38 + umaal r0, lr, r12, r6 + umaal r1, lr, r11, r6 + mov r12, r3 + mov r11, r4 + pop {r2-r4} + umaal r2, lr, r10, r6 + umaal r3, lr, r12, r6 + umaal r4, lr, r11, r6 + mov r12, r6 + pop {r5-r6} + umaal r5, lr, r8, r12 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r7, r7, #0x80000000 +#else + bfc r7, #31, #1 +#endif + umaal r6, lr, r9, r12 + add r7, r7, lr + adds lr, r0, #19 + adcs lr, r1, #0 + adcs lr, r2, #0 + adcs lr, r3, #0 + adcs lr, r4, #0 + adcs lr, r5, #0 + adcs lr, r6, #0 + adc lr, r7, #0 + asr lr, lr, #31 + and lr, lr, #19 + adds r0, r0, lr + adcs r1, r1, #0 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adc r7, r7, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r7, r7, #0x80000000 +#else + bfc r7, #31, #1 +#endif + pop {lr} + # Store + stm lr, {r0, r1, r2, r3, r4, r5, r6, r7} + pop {pc} + .size fe_sq_op_full_red,.-fe_sq_op_full_red #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ .text .align 4 @@ -3462,17 +4475,17 @@ L_curve25519_bits: ldr r2, [sp, #160] add r1, sp, #0x60 add r0, sp, #32 - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x80 mov r1, sp mov r0, sp - bl fe_mul_op + bl fe_mul_op_full_red add r1, sp, #0x80 add r0, sp, #0x80 - bl fe_sq_op + bl fe_sq_op_full_red ldr r1, [sp, #160] add r0, sp, #0x60 - bl fe_sq_op + bl fe_sq_op_full_red mov r3, sp add r2, sp, #32 mov r1, sp @@ -3481,7 +4494,7 @@ L_curve25519_bits: add r2, sp, #0x80 add r1, sp, #0x60 ldr r0, [sp, #160] - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x80 add r1, sp, #0x60 add r0, sp, #0x60 @@ -3494,7 +4507,7 @@ L_curve25519_bits: bl fe_mul121666 add r1, sp, #0x40 add r0, sp, #0x40 - bl fe_sq_op + bl fe_sq_op_full_red add r2, sp, #32 add r1, sp, #0x80 add r0, sp, #0x80 @@ -3502,11 +4515,11 @@ L_curve25519_bits: mov r2, sp ldr r1, [sp, #168] add r0, sp, #32 - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x80 add r1, sp, #0x60 mov r0, sp - bl fe_mul_op + bl fe_mul_op_full_red ldr r2, [sp, #176] ldr r1, [sp, #180] subs r1, r1, #1 @@ -3771,17 +4784,17 @@ L_curve25519_bits: ldr r2, [sp, #176] add r1, sp, #0x60 ldr r0, [sp, #188] - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x80 ldr r1, [sp, #184] ldr r0, [sp, #184] - bl fe_mul_op + bl fe_mul_op_full_red add r1, sp, #0x80 add r0, sp, #0x60 - bl fe_sq_op + bl fe_sq_op_full_red ldr r1, [sp, #176] add r0, sp, #0x80 - bl fe_sq_op + bl fe_sq_op_full_red ldr r3, [sp, #184] ldr r2, [sp, #188] ldr r1, [sp, #184] @@ -3790,7 +4803,7 @@ L_curve25519_bits: add r2, sp, #0x60 add r1, sp, #0x80 ldr r0, [sp, #176] - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x60 add r1, sp, #0x80 add r0, sp, #0x80 @@ -3803,7 +4816,7 @@ L_curve25519_bits: bl fe_mul121666 ldr r1, [sp, #180] ldr r0, [sp, #180] - bl fe_sq_op + bl fe_sq_op_full_red ldr r2, [sp, #188] add r1, sp, #0x60 add r0, sp, #0x60 @@ -3811,11 +4824,11 @@ L_curve25519_bits: ldr r2, [sp, #184] ldr r1, [sp, #172] ldr r0, [sp, #188] - bl fe_mul_op + bl fe_mul_op_full_red add r2, sp, #0x60 add r1, sp, #0x80 ldr r0, [sp, #184] - bl fe_mul_op + bl fe_mul_op_full_red ldr r2, [sp, #168] subs r2, r2, #1 bge L_curve25519_bits @@ -4473,6 +5486,29 @@ fe_sq2: #endif adcs r7, r7, #0 adc r8, r8, #0 + adds r11, r1, #19 + adcs r11, r2, #0 + adcs r11, r3, #0 + adcs r11, r4, #0 + adcs r11, r5, #0 + adcs r11, r6, #0 + adcs r11, r7, #0 + adc r11, r8, #0 + asr r11, r11, #31 + and r11, r11, #19 + adds r1, r1, r11 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adcs r7, r7, #0 + adc r8, r8, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r8, r8, #0x80000000 +#else + bfc r8, #31, #1 +#endif # Store ldr r0, [sp, #64] stm r0, {r1, r2, r3, r4, r5, r6, r7, r8} @@ -4647,6 +5683,29 @@ fe_sq2: #endif adcs r6, r6, #0 adc r7, r7, #0 + adds r12, r0, #19 + adcs r12, r1, #0 + adcs r12, r2, #0 + adcs r12, r3, #0 + adcs r12, r4, #0 + adcs r12, r5, #0 + adcs r12, r6, #0 + adc r12, r7, #0 + asr r12, r12, #31 + and r12, r12, #19 + adds r0, r0, r12 + adcs r1, r1, #0 + adcs r2, r2, #0 + adcs r3, r3, #0 + adcs r4, r4, #0 + adcs r5, r5, #0 + adcs r6, r6, #0 + adc r7, r7, #0 +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + bic r7, r7, #0x80000000 +#else + bfc r7, #31, #1 +#endif pop {r12, lr} # Store stm r12, {r0, r1, r2, r3, r4, r5, r6, r7} diff --git a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c index 7b4925b893..1aa911c249 100644 --- a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c @@ -59,9 +59,9 @@ #if !defined(CURVE25519_SMALL) || !defined(ED25519_SMALL) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_init() +WC_OMIT_FRAME_POINTER void fe_init(void) #else -WC_OMIT_FRAME_POINTER void fe_init() +WC_OMIT_FRAME_POINTER void fe_init(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -81,9 +81,9 @@ WC_OMIT_FRAME_POINTER void fe_init() void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op() +WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op() +WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -296,9 +296,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op() void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op() +WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op() +WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -372,9 +372,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op() +WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op() +WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2498,9 +2498,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2890,12 +2890,429 @@ WC_OMIT_FRAME_POINTER void fe_mul_op() ); } +void fe_mul_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "sub sp, sp, #40\n\t" + "str r0, [sp, #36]\n\t" + "mov r0, #0\n\t" + "ldr r12, [r1]\n\t" + /* A[0] * B[0] */ + "ldr lr, [r2]\n\t" + "umull r3, r4, r12, lr\n\t" + /* A[0] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "umull r5, r6, r12, lr\n\t" + /* A[0] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "umull r7, r8, r12, lr\n\t" + /* A[0] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "umull r9, r10, r12, lr\n\t" + "str r3, [sp]\n\t" + /* A[0] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "mov r11, r0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[0] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adcs r6, r6, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[0] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adcs r8, r8, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[0] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adcs r10, r10, #0\n\t" + "adc r3, r0, #0\n\t" + "umlal r10, r3, r12, lr\n\t" + /* A[1] * B[0] */ + "ldr r12, [r1, #4]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "str r4, [sp, #4]\n\t" + "adds r5, r5, r11\n\t" + /* A[1] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[1] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[1] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[1] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[1] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[1] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[1] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r4, r0, #0\n\t" + "umlal r3, r4, r12, lr\n\t" + /* A[2] * B[0] */ + "ldr r12, [r1, #8]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "str r5, [sp, #8]\n\t" + "adds r6, r6, r11\n\t" + /* A[2] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[2] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[2] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[2] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[2] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[2] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[2] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r5, r0, #0\n\t" + "umlal r4, r5, r12, lr\n\t" + /* A[3] * B[0] */ + "ldr r12, [r1, #12]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "str r6, [sp, #12]\n\t" + "adds r7, r7, r11\n\t" + /* A[3] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[3] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[3] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[3] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[3] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[3] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[3] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r6, r0, #0\n\t" + "umlal r5, r6, r12, lr\n\t" + /* A[4] * B[0] */ + "ldr r12, [r1, #16]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "str r7, [sp, #16]\n\t" + "adds r8, r8, r11\n\t" + /* A[4] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[4] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[4] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[4] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[4] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[4] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[4] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r7, r0, #0\n\t" + "umlal r6, r7, r12, lr\n\t" + /* A[5] * B[0] */ + "ldr r12, [r1, #20]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "str r8, [sp, #20]\n\t" + "adds r9, r9, r11\n\t" + /* A[5] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[5] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[5] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[5] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[5] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[5] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[5] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r8, r0, #0\n\t" + "umlal r7, r8, r12, lr\n\t" + /* A[6] * B[0] */ + "ldr r12, [r1, #24]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "str r9, [sp, #24]\n\t" + "adds r10, r10, r11\n\t" + /* A[6] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[6] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[6] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[6] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[6] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[6] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[6] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r9, r0, #0\n\t" + "umlal r8, r9, r12, lr\n\t" + /* A[7] * B[0] */ + "ldr r12, [r1, #28]\n\t" + "ldr lr, [r2]\n\t" + "mov r11, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "str r10, [sp, #28]\n\t" + "adds r3, r3, r11\n\t" + /* A[7] * B[1] */ + "ldr lr, [r2, #4]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[7] * B[2] */ + "ldr lr, [r2, #8]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[7] * B[3] */ + "ldr lr, [r2, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[7] * B[4] */ + "ldr lr, [r2, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[7] * B[5] */ + "ldr lr, [r2, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[7] * B[6] */ + "ldr lr, [r2, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[7] * B[7] */ + "ldr lr, [r2, #28]\n\t" + "adc r10, r0, #0\n\t" + "umlal r9, r10, r12, lr\n\t" + /* Reduce */ + "ldr r2, [sp, #28]\n\t" + "mov lr, sp\n\t" + "mov r12, #38\n\t" + "umull r10, r11, r12, r10\n\t" + "adds r10, r10, r2\n\t" + "adc r11, r11, #0\n\t" + "mov r12, #19\n\t" + "lsl r11, r11, #1\n\t" + "orr r11, r11, r10, lsr #31\n\t" + "mul r11, r12, r11\n\t" + "ldm lr!, {r1, r2}\n\t" + "mov r12, #38\n\t" + "adds r1, r1, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r1, r11, r3, r12\n\t" + "adds r2, r2, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r2, r11, r4, r12\n\t" + "ldm lr!, {r3, r4}\n\t" + "adds r3, r3, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r5, r12\n\t" + "adds r4, r4, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r6, r12\n\t" + "ldm lr!, {r5, r6}\n\t" + "adds r5, r5, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r7, r12\n\t" + "adds r6, r6, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r8, r12\n\t" + "ldm lr!, {r7, r8}\n\t" + "adds r7, r7, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r9, r12\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r10, r10, #0x80000000\n\t" +#else + "bfc r10, #31, #1\n\t" +#endif + "adds r8, r10, r11\n\t" + "adds r11, r1, #19\n\t" + "adcs r11, r2, #0\n\t" + "adcs r11, r3, #0\n\t" + "adcs r11, r4, #0\n\t" + "adcs r11, r5, #0\n\t" + "adcs r11, r6, #0\n\t" + "adcs r11, r7, #0\n\t" + "adc r11, r8, #0\n\t" + "asr r11, r11, #31\n\t" + "and r11, r11, #19\n\t" + "adds r1, r1, r11\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adcs r7, r7, #0\n\t" + "adc r8, r8, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r8, r8, #0x80000000\n\t" +#else + "bfc r8, #31, #1\n\t" +#endif + /* Store */ + "ldr r0, [sp, #36]\n\t" + "stm r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" + "add sp, sp, #40\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3044,6 +3461,182 @@ WC_OMIT_FRAME_POINTER void fe_mul_op() ); } +void fe_mul_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "sub sp, sp, #44\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "str r0, [sp, #36]\n\t" + "str r1, [sp, #40]\n\t" +#else + "strd r0, r1, [sp, #36]\n\t" +#endif + "mov lr, r2\n\t" + "ldm r1, {r0, r1, r2, r3}\n\t" + "ldm lr!, {r4, r5, r6}\n\t" + "umull r10, r11, r0, r4\n\t" + "umull r12, r7, r1, r4\n\t" + "umaal r11, r12, r0, r5\n\t" + "umull r8, r9, r2, r4\n\t" + "umaal r12, r8, r1, r5\n\t" + "umaal r12, r7, r0, r6\n\t" + "umaal r8, r9, r3, r4\n\t" + "stm sp, {r10, r11, r12}\n\t" + "umaal r7, r8, r2, r5\n\t" + "ldm lr!, {r4}\n\t" + "umull r10, r11, r1, r6\n\t" + "umaal r8, r9, r2, r6\n\t" + "umaal r7, r10, r0, r4\n\t" + "umaal r8, r11, r3, r5\n\t" + "str r7, [sp, #12]\n\t" + "umaal r8, r10, r1, r4\n\t" + "umaal r9, r11, r3, r6\n\t" + "umaal r9, r10, r2, r4\n\t" + "umaal r10, r11, r3, r4\n\t" + "ldm lr, {r4, r5, r6, r7}\n\t" + "mov r12, #0\n\t" + "umlal r8, r12, r0, r4\n\t" + "umaal r9, r12, r1, r4\n\t" + "umaal r10, r12, r2, r4\n\t" + "umaal r11, r12, r3, r4\n\t" + "mov r4, #0\n\t" + "umlal r9, r4, r0, r5\n\t" + "umaal r10, r4, r1, r5\n\t" + "umaal r11, r4, r2, r5\n\t" + "umaal r12, r4, r3, r5\n\t" + "mov r5, #0\n\t" + "umlal r10, r5, r0, r6\n\t" + "umaal r11, r5, r1, r6\n\t" + "umaal r12, r5, r2, r6\n\t" + "umaal r4, r5, r3, r6\n\t" + "mov r6, #0\n\t" + "umlal r11, r6, r0, r7\n\t" + "ldr r0, [sp, #40]\n\t" + "umaal r12, r6, r1, r7\n\t" + "add r0, r0, #16\n\t" + "umaal r4, r6, r2, r7\n\t" + "sub lr, lr, #16\n\t" + "umaal r5, r6, r3, r7\n\t" + "ldm r0, {r0, r1, r2, r3}\n\t" + "str r6, [sp, #32]\n\t" + "ldm lr!, {r6}\n\t" + "mov r7, #0\n\t" + "umlal r8, r7, r0, r6\n\t" + "umaal r9, r7, r1, r6\n\t" + "str r8, [sp, #16]\n\t" + "umaal r10, r7, r2, r6\n\t" + "umaal r11, r7, r3, r6\n\t" + "ldm lr!, {r6}\n\t" + "mov r8, #0\n\t" + "umlal r9, r8, r0, r6\n\t" + "umaal r10, r8, r1, r6\n\t" + "str r9, [sp, #20]\n\t" + "umaal r11, r8, r2, r6\n\t" + "umaal r12, r8, r3, r6\n\t" + "ldm lr!, {r6}\n\t" + "mov r9, #0\n\t" + "umlal r10, r9, r0, r6\n\t" + "umaal r11, r9, r1, r6\n\t" + "str r10, [sp, #24]\n\t" + "umaal r12, r9, r2, r6\n\t" + "umaal r4, r9, r3, r6\n\t" + "ldm lr!, {r6}\n\t" + "mov r10, #0\n\t" + "umlal r11, r10, r0, r6\n\t" + "umaal r12, r10, r1, r6\n\t" + "str r11, [sp, #28]\n\t" + "umaal r4, r10, r2, r6\n\t" + "umaal r5, r10, r3, r6\n\t" + "ldm lr!, {r11}\n\t" + "umaal r12, r7, r0, r11\n\t" + "umaal r4, r7, r1, r11\n\t" + "ldr r6, [sp, #32]\n\t" + "umaal r5, r7, r2, r11\n\t" + "umaal r6, r7, r3, r11\n\t" + "ldm lr!, {r11}\n\t" + "umaal r4, r8, r0, r11\n\t" + "umaal r5, r8, r1, r11\n\t" + "umaal r6, r8, r2, r11\n\t" + "umaal r7, r8, r3, r11\n\t" + "ldm lr, {r11, lr}\n\t" + "umaal r5, r9, r0, r11\n\t" + "umaal r6, r10, r0, lr\n\t" + "umaal r6, r9, r1, r11\n\t" + "umaal r7, r10, r1, lr\n\t" + "umaal r7, r9, r2, r11\n\t" + "umaal r8, r10, r2, lr\n\t" + "umaal r8, r9, r3, r11\n\t" + "umaal r9, r10, r3, lr\n\t" + /* Reduce */ + "ldr r0, [sp, #28]\n\t" + "mov lr, #37\n\t" + "umaal r10, r0, r10, lr\n\t" + "mov lr, #19\n\t" + "lsl r0, r0, #1\n\t" + "orr r0, r0, r10, lsr #31\n\t" + "mul r11, r0, lr\n\t" + "pop {r0-r2}\n\t" + "mov lr, #38\n\t" + "umaal r0, r11, r12, lr\n\t" + "umaal r1, r11, r4, lr\n\t" + "umaal r2, r11, r5, lr\n\t" + "pop {r3-r5}\n\t" + "umaal r3, r11, r6, lr\n\t" + "umaal r4, r11, r7, lr\n\t" + "umaal r5, r11, r8, lr\n\t" + "pop {r6}\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r10, r10, #0x80000000\n\t" +#else + "bfc r10, #31, #1\n\t" +#endif + "umaal r6, r11, r9, lr\n\t" + "add r7, r10, r11\n\t" + "adds r11, r0, #19\n\t" + "adcs r11, r1, #0\n\t" + "adcs r11, r2, #0\n\t" + "adcs r11, r3, #0\n\t" + "adcs r11, r4, #0\n\t" + "adcs r11, r5, #0\n\t" + "adcs r11, r6, #0\n\t" + "adc r11, r7, #0\n\t" + "asr r11, r11, #31\n\t" + "and r11, r11, #19\n\t" + "adds r0, r0, r11\n\t" + "adcs r1, r1, #0\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r7, r7, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r7, r7, #0x80000000\n\t" +#else + "bfc r7, #31, #1\n\t" +#endif + "ldr lr, [sp, #8]\n\t" + /* Store */ + "stm lr, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" + "add sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ #ifndef WOLFSSL_NO_VAR_ASSIGN_REG WC_OMIT_FRAME_POINTER void fe_mul(fe r_p, const fe a_p, const fe b_p) @@ -3074,9 +3667,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3359,12 +3952,322 @@ WC_OMIT_FRAME_POINTER void fe_sq_op() ); } +void fe_sq_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "sub sp, sp, #0x44\n\t" + "str r0, [sp, #64]\n\t" + /* Square */ + "mov r0, #0\n\t" + "ldr r12, [r1]\n\t" + /* A[0] * A[1] */ + "ldr lr, [r1, #4]\n\t" + "umull r4, r5, r12, lr\n\t" + /* A[0] * A[3] */ + "ldr lr, [r1, #12]\n\t" + "umull r6, r7, r12, lr\n\t" + /* A[0] * A[5] */ + "ldr lr, [r1, #20]\n\t" + "umull r8, r9, r12, lr\n\t" + /* A[0] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "umull r10, r3, r12, lr\n\t" + /* A[0] * A[2] */ + "ldr lr, [r1, #8]\n\t" + "mov r11, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[0] * A[4] */ + "ldr lr, [r1, #16]\n\t" + "adcs r7, r7, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "adds r8, r8, r11\n\t" + /* A[0] * A[6] */ + "ldr lr, [r1, #24]\n\t" + "adcs r9, r9, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + "adcs r3, r3, #0\n\t" + "str r4, [sp, #4]\n\t" + "str r5, [sp, #8]\n\t" + /* A[1] * A[2] */ + "ldr r12, [r1, #4]\n\t" + "ldr lr, [r1, #8]\n\t" + "mov r11, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "str r6, [sp, #12]\n\t" + "adds r7, r7, r11\n\t" + /* A[1] * A[3] */ + "ldr lr, [r1, #12]\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, lr\n\t" + "str r7, [sp, #16]\n\t" + "adds r8, r8, r11\n\t" + /* A[1] * A[4] */ + "ldr lr, [r1, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "adds r9, r9, r11\n\t" + /* A[1] * A[5] */ + "ldr lr, [r1, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "adds r10, r10, r11\n\t" + /* A[1] * A[6] */ + "ldr lr, [r1, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[1] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "adc r4, r0, #0\n\t" + "umlal r3, r4, r12, lr\n\t" + /* A[2] * A[3] */ + "ldr r12, [r1, #8]\n\t" + "ldr lr, [r1, #12]\n\t" + "mov r11, #0\n\t" + "umlal r8, r11, r12, lr\n\t" + "str r8, [sp, #20]\n\t" + "adds r9, r9, r11\n\t" + /* A[2] * A[4] */ + "ldr lr, [r1, #16]\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, lr\n\t" + "str r9, [sp, #24]\n\t" + "adds r10, r10, r11\n\t" + /* A[2] * A[5] */ + "ldr lr, [r1, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "adds r3, r3, r11\n\t" + /* A[2] * A[6] */ + "ldr lr, [r1, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[2] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "adc r5, r0, #0\n\t" + "umlal r4, r5, r12, lr\n\t" + /* A[3] * A[4] */ + "ldr r12, [r1, #12]\n\t" + "ldr lr, [r1, #16]\n\t" + "mov r11, #0\n\t" + "umlal r10, r11, r12, lr\n\t" + "str r10, [sp, #28]\n\t" + "adds r3, r3, r11\n\t" + /* A[3] * A[5] */ + "ldr lr, [r1, #20]\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, lr\n\t" + "adds r4, r4, r11\n\t" + /* A[3] * A[6] */ + "ldr lr, [r1, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[3] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "adc r6, r0, #0\n\t" + "umlal r5, r6, r12, lr\n\t" + /* A[4] * A[5] */ + "ldr r12, [r1, #16]\n\t" + "ldr lr, [r1, #20]\n\t" + "mov r11, #0\n\t" + "umlal r4, r11, r12, lr\n\t" + "adds r5, r5, r11\n\t" + /* A[4] * A[6] */ + "ldr lr, [r1, #24]\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, lr\n\t" + "adds r6, r6, r11\n\t" + /* A[4] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "adc r7, r0, #0\n\t" + "umlal r6, r7, r12, lr\n\t" + /* A[5] * A[6] */ + "ldr r12, [r1, #20]\n\t" + "ldr lr, [r1, #24]\n\t" + "mov r11, #0\n\t" + "umlal r6, r11, r12, lr\n\t" + "adds r7, r7, r11\n\t" + /* A[5] * A[7] */ + "ldr lr, [r1, #28]\n\t" + "adc r8, r0, #0\n\t" + "umlal r7, r8, r12, lr\n\t" + /* A[6] * A[7] */ + "ldr r12, [r1, #24]\n\t" + "ldr lr, [r1, #28]\n\t" + "mov r9, #0\n\t" + "umlal r8, r9, r12, lr\n\t" + "add lr, sp, #32\n\t" + "stm lr, {r3, r4, r5, r6, r7, r8, r9}\n\t" + "add lr, sp, #4\n\t" + "ldm lr, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "adds r4, r4, r4\n\t" + "adcs r5, r5, r5\n\t" + "adcs r6, r6, r6\n\t" + "adcs r7, r7, r7\n\t" + "adcs r8, r8, r8\n\t" + "adcs r9, r9, r9\n\t" + "adcs r10, r10, r10\n\t" + "stm lr!, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "ldm lr, {r3, r4, r5, r6, r7, r8, r9}\n\t" + "adcs r3, r3, r3\n\t" + "adcs r4, r4, r4\n\t" + "adcs r5, r5, r5\n\t" + "adcs r6, r6, r6\n\t" + "adcs r7, r7, r7\n\t" + "adcs r8, r8, r8\n\t" + "adcs r9, r9, r9\n\t" + "adc r10, r0, #0\n\t" + "stm lr, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + "add lr, sp, #4\n\t" + "ldm lr, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "mov lr, sp\n\t" + /* A[0] * A[0] */ + "ldr r12, [r1]\n\t" + "umull r3, r11, r12, r12\n\t" + "adds r4, r4, r11\n\t" + /* A[1] * A[1] */ + "ldr r12, [r1, #4]\n\t" + "adcs r5, r5, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, r12\n\t" + "adds r6, r6, r11\n\t" + /* A[2] * A[2] */ + "ldr r12, [r1, #8]\n\t" + "adcs r7, r7, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, r12\n\t" + "adds r8, r8, r11\n\t" + /* A[3] * A[3] */ + "ldr r12, [r1, #12]\n\t" + "adcs r9, r9, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r9, r11, r12, r12\n\t" + "adds r10, r10, r11\n\t" + "stm lr!, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + "ldm lr, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + /* A[4] * A[4] */ + "ldr r12, [r1, #16]\n\t" + "adcs r3, r3, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r12, r12\n\t" + "adds r4, r4, r11\n\t" + /* A[5] * A[5] */ + "ldr r12, [r1, #20]\n\t" + "adcs r5, r5, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r12, r12\n\t" + "adds r6, r6, r11\n\t" + /* A[6] * A[6] */ + "ldr r12, [r1, #24]\n\t" + "adcs r7, r7, #0\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r12, r12\n\t" + "adds r8, r8, r11\n\t" + /* A[7] * A[7] */ + "ldr r12, [r1, #28]\n\t" + "adcs r9, r9, #0\n\t" + "adc r10, r10, #0\n\t" + "umlal r9, r10, r12, r12\n\t" + /* Reduce */ + "ldr r2, [sp, #28]\n\t" + "mov lr, sp\n\t" + "mov r12, #38\n\t" + "umull r10, r11, r12, r10\n\t" + "adds r10, r10, r2\n\t" + "adc r11, r11, #0\n\t" + "mov r12, #19\n\t" + "lsl r11, r11, #1\n\t" + "orr r11, r11, r10, lsr #31\n\t" + "mul r11, r12, r11\n\t" + "ldm lr!, {r1, r2}\n\t" + "mov r12, #38\n\t" + "adds r1, r1, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r1, r11, r3, r12\n\t" + "adds r2, r2, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r2, r11, r4, r12\n\t" + "ldm lr!, {r3, r4}\n\t" + "adds r3, r3, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r3, r11, r5, r12\n\t" + "adds r4, r4, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r4, r11, r6, r12\n\t" + "ldm lr!, {r5, r6}\n\t" + "adds r5, r5, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r5, r11, r7, r12\n\t" + "adds r6, r6, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r6, r11, r8, r12\n\t" + "ldm lr!, {r7, r8}\n\t" + "adds r7, r7, r11\n\t" + "adc r11, r0, #0\n\t" + "umlal r7, r11, r9, r12\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r10, r10, #0x80000000\n\t" +#else + "bfc r10, #31, #1\n\t" +#endif + "adds r8, r10, r11\n\t" + "adds r11, r1, #19\n\t" + "adcs r11, r2, #0\n\t" + "adcs r11, r3, #0\n\t" + "adcs r11, r4, #0\n\t" + "adcs r11, r5, #0\n\t" + "adcs r11, r6, #0\n\t" + "adcs r11, r7, #0\n\t" + "adc r11, r8, #0\n\t" + "asr r11, r11, #31\n\t" + "and r11, r11, #19\n\t" + "adds r1, r1, r11\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adcs r7, r7, #0\n\t" + "adc r8, r8, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r8, r8, #0x80000000\n\t" +#else + "bfc r8, #31, #1\n\t" +#endif + /* Store */ + "ldr r0, [sp, #64]\n\t" + "stm r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" + "add sp, sp, #0x44\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3499,6 +4402,168 @@ WC_OMIT_FRAME_POINTER void fe_sq_op() ); } +void fe_sq_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "sub sp, sp, #32\n\t" + "str r0, [sp, #28]\n\t" + "ldm r1, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" + /* Square */ + "umull r9, r10, r0, r0\n\t" + "umull r11, r12, r0, r1\n\t" + "adds r11, r11, r11\n\t" + "mov lr, #0\n\t" + "umaal r10, r11, lr, lr\n\t" + "stm sp, {r9, r10}\n\t" + "mov r8, lr\n\t" + "umaal r8, r12, r0, r2\n\t" + "adcs r8, r8, r8\n\t" + "umaal r8, r11, r1, r1\n\t" + "umull r9, r10, r0, r3\n\t" + "umaal r9, r12, r1, r2\n\t" + "adcs r9, r9, r9\n\t" + "umaal r9, r11, lr, lr\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "str r8, [sp, #8]\n\t" + "str r9, [sp, #12]\n\t" +#else + "strd r8, r9, [sp, #8]\n\t" +#endif + "mov r9, lr\n\t" + "umaal r9, r10, r0, r4\n\t" + "umaal r9, r12, r1, r3\n\t" + "adcs r9, r9, r9\n\t" + "umaal r9, r11, r2, r2\n\t" + "str r9, [sp, #16]\n\t" + "umull r9, r8, r0, r5\n\t" + "umaal r9, r12, r1, r4\n\t" + "umaal r9, r10, r2, r3\n\t" + "adcs r9, r9, r9\n\t" + "umaal r9, r11, lr, lr\n\t" + "str r9, [sp, #20]\n\t" + "mov r9, lr\n\t" + "umaal r9, r8, r0, r6\n\t" + "umaal r9, r12, r1, r5\n\t" + "umaal r9, r10, r2, r4\n\t" + "adcs r9, r9, r9\n\t" + "umaal r9, r11, r3, r3\n\t" + "str r9, [sp, #24]\n\t" + "umull r0, r9, r0, r7\n\t" + "umaal r0, r8, r1, r6\n\t" + "umaal r0, r12, r2, r5\n\t" + "umaal r0, r10, r3, r4\n\t" + "adcs r0, r0, r0\n\t" + "umaal r0, r11, lr, lr\n\t" + /* R[7] = r0 */ + "umaal r9, r8, r1, r7\n\t" + "umaal r9, r10, r2, r6\n\t" + "umaal r12, r9, r3, r5\n\t" + "adcs r12, r12, r12\n\t" + "umaal r12, r11, r4, r4\n\t" + /* R[8] = r12 */ + "umaal r9, r8, r2, r7\n\t" + "umaal r10, r9, r3, r6\n\t" + "mov r2, lr\n\t" + "umaal r10, r2, r4, r5\n\t" + "adcs r10, r10, r10\n\t" + "umaal r11, r10, lr, lr\n\t" + /* R[9] = r11 */ + "umaal r2, r8, r3, r7\n\t" + "umaal r2, r9, r4, r6\n\t" + "adcs r3, r2, r2\n\t" + "umaal r10, r3, r5, r5\n\t" + /* R[10] = r10 */ + "mov r1, lr\n\t" + "umaal r1, r8, r4, r7\n\t" + "umaal r1, r9, r5, r6\n\t" + "adcs r4, r1, r1\n\t" + "umaal r3, r4, lr, lr\n\t" + /* R[11] = r3 */ + "umaal r8, r9, r5, r7\n\t" + "adcs r8, r8, r8\n\t" + "umaal r4, r8, r6, r6\n\t" + /* R[12] = r4 */ + "mov r5, lr\n\t" + "umaal r5, r9, r6, r7\n\t" + "adcs r5, r5, r5\n\t" + "umaal r8, r5, lr, lr\n\t" + /* R[13] = r8 */ + "adcs r9, r9, r9\n\t" + "umaal r9, r5, r7, r7\n\t" + "adcs r7, r5, lr\n\t" + /* R[14] = r9 */ + /* R[15] = r7 */ + /* Reduce */ + "mov r6, #37\n\t" + "umaal r7, r0, r7, r6\n\t" + "mov r6, #19\n\t" + "lsl r0, r0, #1\n\t" + "orr r0, r0, r7, lsr #31\n\t" + "mul lr, r0, r6\n\t" + "pop {r0-r1}\n\t" + "mov r6, #38\n\t" + "umaal r0, lr, r12, r6\n\t" + "umaal r1, lr, r11, r6\n\t" + "mov r12, r3\n\t" + "mov r11, r4\n\t" + "pop {r2-r4}\n\t" + "umaal r2, lr, r10, r6\n\t" + "umaal r3, lr, r12, r6\n\t" + "umaal r4, lr, r11, r6\n\t" + "mov r12, r6\n\t" + "pop {r5-r6}\n\t" + "umaal r5, lr, r8, r12\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r7, r7, #0x80000000\n\t" +#else + "bfc r7, #31, #1\n\t" +#endif + "umaal r6, lr, r9, r12\n\t" + "add r7, r7, lr\n\t" + "adds lr, r0, #19\n\t" + "adcs lr, r1, #0\n\t" + "adcs lr, r2, #0\n\t" + "adcs lr, r3, #0\n\t" + "adcs lr, r4, #0\n\t" + "adcs lr, r5, #0\n\t" + "adcs lr, r6, #0\n\t" + "adc lr, r7, #0\n\t" + "asr lr, lr, #31\n\t" + "and lr, lr, #19\n\t" + "adds r0, r0, lr\n\t" + "adcs r1, r1, #0\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r7, r7, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r7, r7, #0x80000000\n\t" +#else + "bfc r7, #31, #1\n\t" +#endif + "pop {lr}\n\t" + /* Store */ + "stm lr, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #endif /* WOLFSSL_ARM_ARCH && WOLFSSL_ARM_ARCH < 6 */ #ifndef WOLFSSL_NO_VAR_ASSIGN_REG WC_OMIT_FRAME_POINTER void fe_sq(fe r_p, const fe a_p) @@ -3859,17 +4924,17 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "ldr r2, [sp, #160]\n\t" "add r1, sp, #0x60\n\t" "add r0, sp, #32\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x80\n\t" "mov r1, sp\n\t" "mov r0, sp\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r1, sp, #0x80\n\t" "add r0, sp, #0x80\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "ldr r1, [sp, #160]\n\t" "add r0, sp, #0x60\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "mov r3, sp\n\t" "add r2, sp, #32\n\t" "mov r1, sp\n\t" @@ -3878,7 +4943,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "add r2, sp, #0x80\n\t" "add r1, sp, #0x60\n\t" "ldr r0, [sp, #160]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x80\n\t" "add r1, sp, #0x60\n\t" "add r0, sp, #0x60\n\t" @@ -3891,7 +4956,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "bl fe_mul121666\n\t" "add r1, sp, #0x40\n\t" "add r0, sp, #0x40\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "add r2, sp, #32\n\t" "add r1, sp, #0x80\n\t" "add r0, sp, #0x80\n\t" @@ -3899,11 +4964,11 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "mov r2, sp\n\t" "ldr r1, [sp, #168]\n\t" "add r0, sp, #32\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x80\n\t" "add r1, sp, #0x60\n\t" "mov r0, sp\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "ldr %[a], [sp, #176]\n\t" "ldr %[n], [sp, #180]\n\t" "subs %[n], %[n], #1\n\t" @@ -4196,17 +5261,17 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "ldr r2, [sp, #176]\n\t" "add r1, sp, #0x60\n\t" "ldr r0, [sp, #188]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x80\n\t" "ldr r1, [sp, #184]\n\t" "ldr r0, [sp, #184]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r1, sp, #0x80\n\t" "add r0, sp, #0x60\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "ldr r1, [sp, #176]\n\t" "add r0, sp, #0x80\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "ldr r3, [sp, #184]\n\t" "ldr r2, [sp, #188]\n\t" "ldr r1, [sp, #184]\n\t" @@ -4215,7 +5280,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "add r2, sp, #0x60\n\t" "add r1, sp, #0x80\n\t" "ldr r0, [sp, #176]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x60\n\t" "add r1, sp, #0x80\n\t" "add r0, sp, #0x80\n\t" @@ -4228,7 +5293,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "bl fe_mul121666\n\t" "ldr r1, [sp, #180]\n\t" "ldr r0, [sp, #180]\n\t" - "bl fe_sq_op\n\t" + "bl fe_sq_op_full_red\n\t" "ldr r2, [sp, #188]\n\t" "add r1, sp, #0x60\n\t" "add r0, sp, #0x60\n\t" @@ -4236,11 +5301,11 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "ldr r2, [sp, #184]\n\t" "ldr r1, [sp, #172]\n\t" "ldr r0, [sp, #188]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "add r2, sp, #0x60\n\t" "add r1, sp, #0x80\n\t" "ldr r0, [sp, #184]\n\t" - "bl fe_mul_op\n\t" + "bl fe_mul_op_full_red\n\t" "ldr %[a], [sp, #168]\n\t" "subs %[a], %[a], #1\n\t" "bge L_curve25519_bits_%=\n\t" @@ -4947,6 +6012,29 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif "adcs r7, r7, #0\n\t" "adc r8, r8, #0\n\t" + "adds r11, r1, #19\n\t" + "adcs r11, r2, #0\n\t" + "adcs r11, r3, #0\n\t" + "adcs r11, r4, #0\n\t" + "adcs r11, r5, #0\n\t" + "adcs r11, r6, #0\n\t" + "adcs r11, r7, #0\n\t" + "adc r11, r8, #0\n\t" + "asr r11, r11, #31\n\t" + "and r11, r11, #19\n\t" + "adds r1, r1, r11\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adcs r7, r7, #0\n\t" + "adc r8, r8, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r8, r8, #0x80000000\n\t" +#else + "bfc r8, #31, #1\n\t" +#endif /* Store */ "ldr r0, [sp, #64]\n\t" "stm r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" @@ -5136,6 +6224,29 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif "adcs r6, r6, #0\n\t" "adc r7, r7, #0\n\t" + "adds r12, r0, #19\n\t" + "adcs r12, r1, #0\n\t" + "adcs r12, r2, #0\n\t" + "adcs r12, r3, #0\n\t" + "adcs r12, r4, #0\n\t" + "adcs r12, r5, #0\n\t" + "adcs r12, r6, #0\n\t" + "adc r12, r7, #0\n\t" + "asr r12, r12, #31\n\t" + "and r12, r12, #19\n\t" + "adds r0, r0, r12\n\t" + "adcs r1, r1, #0\n\t" + "adcs r2, r2, #0\n\t" + "adcs r3, r3, #0\n\t" + "adcs r4, r4, #0\n\t" + "adcs r5, r5, #0\n\t" + "adcs r6, r6, #0\n\t" + "adc r7, r7, #0\n\t" +#if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 7) + "bic r7, r7, #0x80000000\n\t" +#else + "bfc r7, #31, #1\n\t" +#endif "pop {r12, lr}\n\t" /* Store */ "stm r12, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.S b/wolfcrypt/src/port/arm/armv8-aes-asm.S index 0909a9eb79..63080e12db 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.S @@ -5736,7 +5736,7 @@ L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done: eor x14, x14, x14 - lsl x24, x4, #3 + ubfiz x24, x4, #3, #32 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b @@ -7099,10 +7099,10 @@ L_aes_gcm_encrypt_arm64_crypto_192_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_192_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -8637,10 +8637,10 @@ L_aes_gcm_encrypt_arm64_crypto_256_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_256_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b @@ -9944,10 +9944,10 @@ L_aes_gcm_encrypt_arm64_crypto_128_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_128_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -10537,7 +10537,7 @@ L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes: # Done GHASH L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done: eor x14, x14, x14 - lsl x24, x4, #3 + ubfiz x24, x4, #3, #32 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b @@ -11893,10 +11893,10 @@ L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_192_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -11945,6 +11945,7 @@ L_aes_gcm_decrypt_arm64_crypto_192_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_192_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -13446,10 +13447,10 @@ L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_256_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b @@ -13506,6 +13507,7 @@ L_aes_gcm_decrypt_arm64_crypto_256_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_256_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -14768,10 +14770,10 @@ L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_128_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -14816,6 +14818,7 @@ L_aes_gcm_decrypt_arm64_crypto_128_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_128_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -15364,7 +15367,7 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done: eor x14, x14, x14 - lsl x24, x4, #3 + ubfiz x24, x4, #3, #32 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b @@ -16698,10 +16701,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -18207,10 +18210,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b @@ -19485,10 +19488,10 @@ L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero: # Done GHASH L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -20056,7 +20059,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes: # Done GHASH L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done: eor x14, x14, x14 - lsl x24, x4, #3 + ubfiz x24, x4, #3, #32 mov v28.d[0], x14 mov v28.d[1], x24 rev64 v28.16b, v28.16b @@ -21383,10 +21386,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -21434,6 +21437,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -22907,10 +22911,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 aese v14.16b, v0.16b @@ -22966,6 +22970,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -24200,10 +24205,10 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte: L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes: L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done: ld1 {v14.2d}, [x12] - lsl x8, x8, #3 + ubfiz x8, x8, #3, #32 rbit x8, x8 mov v28.d[0], x8 - lsl x2, x2, #3 + ubfiz x2, x2, #3, #32 rbit x2, x2 mov v28.d[1], x2 eor v26.16b, v26.16b, v28.16b @@ -24247,6 +24252,7 @@ L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done: ld1 {v28.16b}, [x5] b L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag: + ubfiz x6, x6, #0, #32 eor v28.16b, v28.16b, v28.16b mov x17, x6 st1 {v28.2d}, [x11] @@ -24420,7 +24426,7 @@ L_aes_gcm_init_arm64_crypto_end_bytes: # Done GHASH L_aes_gcm_init_arm64_crypto_partial_done: eor x7, x7, x7 - lsl x13, x3, #3 + ubfiz x13, x3, #3, #32 mov v7.d[0], x7 mov v7.d[1], x13 rev64 v7.16b, v7.16b @@ -28782,10 +28788,10 @@ _AES_GCM_encrypt_final_AARCH64: ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] - lsl x4, x4, #3 + ubfiz x4, x4, #3, #32 rbit x4, x4 mov v0.d[0], x4 - lsl x3, x3, #3 + ubfiz x3, x3, #3, #32 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b @@ -32668,10 +32674,10 @@ _AES_GCM_decrypt_final_AARCH64: ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] - lsl x4, x4, #3 + ubfiz x4, x4, #3, #32 rbit x4, x4 mov v0.d[0], x4 - lsl x3, x3, #3 + ubfiz x3, x3, #3, #32 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b @@ -32696,6 +32702,7 @@ _AES_GCM_decrypt_final_AARCH64: ld1 {v0.16b}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_part_tag: + ubfiz x2, x2, #0, #32 eor v0.16b, v0.16b, v0.16b mov x10, x2 st1 {v0.2d}, [x0] @@ -32856,7 +32863,7 @@ L_aes_gcm_init_arm64_crypto_eor3_end_bytes: # Done GHASH L_aes_gcm_init_arm64_crypto_eor3_partial_done: eor x7, x7, x7 - lsl x13, x3, #3 + ubfiz x13, x3, #3, #32 mov v7.d[0], x7 mov v7.d[1], x13 rev64 v7.16b, v7.16b @@ -37114,10 +37121,10 @@ _AES_GCM_encrypt_final_AARCH64_EOR3: ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] - lsl x4, x4, #3 + ubfiz x4, x4, #3, #32 rbit x4, x4 mov v0.d[0], x4 - lsl x3, x3, #3 + ubfiz x3, x3, #3, #32 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b @@ -40915,10 +40922,10 @@ _AES_GCM_decrypt_final_AARCH64_EOR3: ld1 {v4.2d}, [x5] ushr v6.2d, v6.2d, #56 ld1 {v7.2d}, [x6] - lsl x4, x4, #3 + ubfiz x4, x4, #3, #32 rbit x4, x4 mov v0.d[0], x4 - lsl x3, x3, #3 + ubfiz x3, x3, #3, #32 rbit x3, x3 mov v0.d[1], x3 eor v5.16b, v5.16b, v0.16b @@ -40942,6 +40949,7 @@ _AES_GCM_decrypt_final_AARCH64_EOR3: ld1 {v0.16b}, [x1] b L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag: + ubfiz x2, x2, #0, #32 eor v0.16b, v0.16b, v0.16b mov x10, x2 st1 {v0.2d}, [x0] diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c index a1b727fd2b..618c77881d 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c @@ -3540,8 +3540,6 @@ void AES_CTR_encrypt_AARCH64(const byte* in, byte* out, word32 sz, byte* reg, byte* key, byte* tmp, word32* left, word32 nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" "ld1 {v15.2d}, [%x[reg]]\n\t" @@ -5191,7 +5189,6 @@ void AES_CTR_encrypt_AARCH64(const byte* in, byte* out, word32 sz, byte* reg, "rev x11, x10\n\t" "rev x12, x9\n\t" "stp x11, x12, [%x[reg]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [reg] "+r" (reg), [key] "+r" (key), [tmp] "+r" (tmp), [left] "+r" (left), [nr] "+r" (nr) : [in] "r" (in) @@ -5264,19 +5261,11 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-80]!\n\t" - "add x29, sp, #0\n\t" - "str %w[nr], [sp, #72]\n\t" - "str %x[reg], [sp, #64]\n\t" - "str %x[tmp], [sp, #56]\n\t" - "str %x[gcm_h], [sp, #48]\n\t" - "str %x[key], [sp, #40]\n\t" - "str %w[aadSz], [sp, #32]\n\t" "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" - "ld1 {v22.2d}, [x10]\n\t" - "cmp w8, #0x40\n\t" + "ld1 {v22.2d}, [%x[gcm_h]]\n\t" + "cmp %w[aadSz], #0x40\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #32\n\t" "csetm x17, lt\n\t" @@ -5291,7 +5280,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v31.1q, v29.2d, v27.2d\n\t" "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" - "cmp w8, #0x100\n\t" + "cmp %w[aadSz], #0x100\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x40\n\t" "csetm x17, lt\n\t" @@ -5322,7 +5311,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v30.d[1], v29.d[0]\n\t" "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ - "cmp w8, #0x400\n\t" + "cmp %w[aadSz], #0x400\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x200\n\t" "csetm x17, lt\n\t" @@ -5379,7 +5368,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done */ "\n" "L_aes_gcm_encrypt_arm64_crypto_h_done_%=:\n\t" - "lsr w14, w8, #4\n\t" + "lsr w14, %w[aadSz], #4\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_aad_start_1_%=\n\t" "cmp w14, #16\n\t" @@ -5617,41 +5606,41 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.ne L_aes_gcm_encrypt_arm64_crypto_aad_both_1_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_aad_done_%=:\n\t" - "and w14, w8, #15\n\t" + "and w14, %w[aadSz], #15\n\t" "cbz w14, L_aes_gcm_encrypt_arm64_crypto_aad_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w14\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_aad_start_dw_%=\n\t" "ldr x19, [%x[aad]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_aad_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_aad_start_sw_%=\n\t" "ldr w19, [%x[aad]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_aad_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_aad_start_byte_%=\n\t" "ldrh w19, [%x[aad]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_aad_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_encrypt_arm64_crypto_aad_end_bytes_%=\n\t" "ldrb w19, [%x[aad]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_aad_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_aad_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v26.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -5716,37 +5705,37 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "cbz x24, L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w24\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_nonce_start_dw_%=\n\t" "ldr x19, [%x[nonce]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_nonce_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_nonce_start_sw_%=\n\t" "ldr w19, [%x[nonce]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_nonce_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte_%=\n\t" "ldrh w19, [%x[nonce]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes_%=\n\t" "ldrb w19, [%x[nonce]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_nonce_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_nonce_end_bytes_%=:\n\t" - "sub x11, x11, x24\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x24\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v13.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -5768,7 +5757,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_nonce_partial_done_%=:\n\t" "eor x14, x14, x14\n\t" - "lsl x24, %x[nonceSz], #3\n\t" + "ubfiz x24, %x[nonceSz], #3, #32\n\t" "mov v28.d[0], x14\n\t" "mov v28.d[1], x24\n\t" "rev64 v28.16b, v28.16b\n\t" @@ -5792,9 +5781,9 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "rev w15, w15\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_done_nonce_%=:\n\t" - "st1 {v13.2d}, [x12]\n\t" + "st1 {v13.2d}, [%x[reg]]\n\t" "lsr w14, %w[sz], #4\n\t" - "cmp w13, #12\n\t" + "cmp %w[nr], #12\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_start_128_%=\n\t" "b.gt L_aes_gcm_encrypt_arm64_crypto_start_256_%=\n\t" /* AES_GCM_192 */ @@ -5803,7 +5792,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_192_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -5836,7 +5825,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -5853,7 +5842,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -5870,7 +5859,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -5887,7 +5876,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -5904,7 +5893,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -5921,7 +5910,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -5938,7 +5927,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -5956,7 +5945,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -5973,7 +5962,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -5994,7 +5983,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -6015,7 +6004,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v3.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -6032,7 +6021,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -6057,14 +6046,14 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -6105,7 +6094,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -6135,7 +6124,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -6163,7 +6152,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -6192,7 +6181,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -6220,7 +6209,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -6248,7 +6237,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -6278,7 +6267,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -6306,7 +6295,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v29.16b\n\t" @@ -6331,7 +6320,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -6354,7 +6343,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -6371,7 +6360,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -6388,7 +6377,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -6405,7 +6394,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -6517,10 +6506,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_192_start_1_%=\n\t" @@ -7031,37 +7020,37 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_192_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_192_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_192_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -7091,31 +7080,31 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v14.16b, v11.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_192_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_192_out_start_byte_%=\n\t" @@ -7126,10 +7115,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_192_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_192_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -7150,11 +7139,11 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_192_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -7204,30 +7193,30 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_192_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_192_tag_start_byte_%=\n\t" @@ -7243,7 +7232,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_256_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -7276,7 +7265,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7293,7 +7282,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7310,7 +7299,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7327,7 +7316,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7344,7 +7333,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7361,7 +7350,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7378,7 +7367,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7396,7 +7385,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7413,7 +7402,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7434,7 +7423,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7455,7 +7444,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v3.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7472,7 +7461,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7489,7 +7478,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7506,7 +7495,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7531,14 +7520,14 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -7579,7 +7568,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -7609,7 +7598,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -7637,7 +7626,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -7666,7 +7655,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -7694,7 +7683,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -7722,7 +7711,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -7752,7 +7741,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -7780,7 +7769,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v29.16b\n\t" @@ -7805,7 +7794,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -7828,7 +7817,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7845,7 +7834,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7862,7 +7851,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7879,7 +7868,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -7896,7 +7885,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -7913,7 +7902,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -8025,10 +8014,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9], #16\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]], #16\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_256_start_1_%=\n\t" @@ -8151,7 +8140,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8342,7 +8331,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8483,7 +8472,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v11.16b\n\t" "aesmc v15.16b, v15.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8556,10 +8545,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" @@ -8588,37 +8577,37 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_256_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_256_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_256_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -8647,38 +8636,38 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_256_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_256_out_start_byte_%=\n\t" @@ -8689,10 +8678,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_256_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_256_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -8713,11 +8702,11 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_256_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "aese v14.16b, v0.16b\n\t" @@ -8753,19 +8742,19 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull2 v30.1q, v31.2d, v27.2d\n\t" - "ldr q11, [x9, #-32]\n\t" + "ldr q11, [%x[key], #-32]\n\t" "aese v14.16b, v10.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "mov v28.d[1], v31.d[0]\n\t" - "ldr q12, [x9, #-16]\n\t" + "ldr q12, [%x[key], #-16]\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v28.16b, v30.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "rbit v26.16b, v26.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v26.16b, v26.16b, v14.16b\n\t" @@ -8775,30 +8764,30 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_256_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_256_tag_start_byte_%=\n\t" @@ -8814,7 +8803,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_128_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -8847,7 +8836,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8864,7 +8853,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -8881,7 +8870,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8898,7 +8887,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -8915,7 +8904,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8932,7 +8921,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -8949,7 +8938,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -8967,7 +8956,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -8984,7 +8973,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -9005,7 +8994,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -9034,14 +9023,14 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -9082,7 +9071,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -9112,7 +9101,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -9140,7 +9129,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -9169,7 +9158,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -9197,7 +9186,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -9225,7 +9214,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -9255,7 +9244,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -9283,7 +9272,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v29.16b\n\t" @@ -9308,7 +9297,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -9331,7 +9320,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -9348,7 +9337,7 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -9460,10 +9449,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d}, [x9], #32\n\t" - "ld1 {v10.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d}, [%x[key]], #32\n\t" + "ld1 {v10.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_128_start_1_%=\n\t" @@ -9930,37 +9919,37 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_128_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_128_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_128_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -9986,31 +9975,31 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "eor v14.16b, v14.16b, v10.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_128_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_128_out_start_byte_%=\n\t" @@ -10021,10 +10010,10 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_128_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_128_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -10045,11 +10034,11 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_128_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -10095,30 +10084,30 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_128_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_128_tag_start_byte_%=\n\t" @@ -10127,7 +10116,6 @@ void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_gcm_encrypt_arm64_crypto_done_%=:\n\t" - "ldp x29, x30, [sp], #0x50\n\t" : [out] "+r" (out), [sz] "+r" (sz), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), @@ -10148,19 +10136,11 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-80]!\n\t" - "add x29, sp, #0\n\t" - "str %w[nr], [sp, #72]\n\t" - "str %x[reg], [sp, #64]\n\t" - "str %x[tmp], [sp, #56]\n\t" - "str %x[gcm_h], [sp, #48]\n\t" - "str %x[key], [sp, #40]\n\t" - "str %w[aadSz], [sp, #32]\n\t" "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" - "ld1 {v22.2d}, [x10]\n\t" - "cmp w8, #0x40\n\t" + "ld1 {v22.2d}, [%x[gcm_h]]\n\t" + "cmp %w[aadSz], #0x40\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #32\n\t" "csetm x17, lt\n\t" @@ -10175,7 +10155,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v31.1q, v29.2d, v27.2d\n\t" "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" - "cmp w8, #0x100\n\t" + "cmp %w[aadSz], #0x100\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x40\n\t" "csetm x17, lt\n\t" @@ -10206,7 +10186,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v30.d[1], v29.d[0]\n\t" "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ - "cmp w8, #0x400\n\t" + "cmp %w[aadSz], #0x400\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x200\n\t" "csetm x17, lt\n\t" @@ -10263,7 +10243,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done */ "\n" "L_aes_gcm_decrypt_arm64_crypto_h_done_%=:\n\t" - "lsr w14, w8, #4\n\t" + "lsr w14, %w[aadSz], #4\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_aad_start_1_%=\n\t" "cmp w14, #16\n\t" @@ -10501,41 +10481,41 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.ne L_aes_gcm_decrypt_arm64_crypto_aad_both_1_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_aad_done_%=:\n\t" - "and w14, w8, #15\n\t" + "and w14, %w[aadSz], #15\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_aad_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w14\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_aad_start_dw_%=\n\t" "ldr x19, [%x[aad]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_aad_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_aad_start_sw_%=\n\t" "ldr w19, [%x[aad]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_aad_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_aad_start_byte_%=\n\t" "ldrh w19, [%x[aad]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_aad_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_decrypt_arm64_crypto_aad_end_bytes_%=\n\t" "ldrb w19, [%x[aad]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_aad_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_aad_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v26.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -10600,37 +10580,37 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "cbz x24, L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w24\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_nonce_start_dw_%=\n\t" "ldr x19, [%x[nonce]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_nonce_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_nonce_start_sw_%=\n\t" "ldr w19, [%x[nonce]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_nonce_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte_%=\n\t" "ldrh w19, [%x[nonce]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes_%=\n\t" "ldrb w19, [%x[nonce]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_nonce_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_nonce_end_bytes_%=:\n\t" - "sub x11, x11, x24\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x24\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v13.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -10652,7 +10632,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_decrypt_arm64_crypto_nonce_partial_done_%=:\n\t" "eor x14, x14, x14\n\t" - "lsl x24, %x[nonceSz], #3\n\t" + "ubfiz x24, %x[nonceSz], #3, #32\n\t" "mov v28.d[0], x14\n\t" "mov v28.d[1], x24\n\t" "rev64 v28.16b, v28.16b\n\t" @@ -10676,9 +10656,9 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "rev w15, w15\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_done_nonce_%=:\n\t" - "st1 {v13.2d}, [x12]\n\t" + "st1 {v13.2d}, [%x[reg]]\n\t" "lsr w14, %w[sz], #4\n\t" - "cmp w13, #12\n\t" + "cmp %w[nr], #12\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_start_128_%=\n\t" "b.gt L_aes_gcm_decrypt_arm64_crypto_start_256_%=\n\t" /* AES_GCM_192 */ @@ -10687,7 +10667,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_192_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -10720,7 +10700,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10737,7 +10717,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10754,7 +10734,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10771,7 +10751,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10788,7 +10768,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10805,7 +10785,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10822,7 +10802,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10840,7 +10820,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10857,7 +10837,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10878,7 +10858,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10899,7 +10879,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -10916,7 +10896,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -10941,14 +10921,14 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -10990,7 +10970,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -11020,7 +11000,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -11049,7 +11029,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -11077,7 +11057,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -11105,7 +11085,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -11135,7 +11115,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -11164,7 +11144,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -11191,7 +11171,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v30.16b\n\t" @@ -11216,7 +11196,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -11238,7 +11218,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -11255,7 +11235,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -11272,7 +11252,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -11289,7 +11269,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -11401,10 +11381,10 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_192_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_192_start_1_%=\n\t" @@ -11916,37 +11896,37 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_192_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_192_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_192_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -11995,30 +11975,30 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_192_out_start_byte_%=\n\t" @@ -12026,11 +12006,11 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_192_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -12080,50 +12060,51 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_192_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_192_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_192_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_192_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -12144,7 +12125,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_256_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -12177,7 +12158,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12194,7 +12175,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12211,7 +12192,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12228,7 +12209,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12245,7 +12226,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12262,7 +12243,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12279,7 +12260,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12297,7 +12278,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12314,7 +12295,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12335,7 +12316,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12356,7 +12337,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12373,7 +12354,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12390,7 +12371,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12407,7 +12388,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12432,14 +12413,14 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -12481,7 +12462,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -12511,7 +12492,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -12540,7 +12521,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -12568,7 +12549,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -12596,7 +12577,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -12626,7 +12607,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -12655,7 +12636,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -12682,7 +12663,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v30.16b\n\t" @@ -12707,7 +12688,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -12729,7 +12710,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12746,7 +12727,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12763,7 +12744,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12780,7 +12761,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -12797,7 +12778,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -12814,7 +12795,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -12926,10 +12907,10 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_256_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9], #16\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]], #16\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_256_start_1_%=\n\t" @@ -13052,7 +13033,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13243,7 +13224,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13384,7 +13365,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v11.16b\n\t" "aesmc v15.16b, v15.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13457,10 +13438,10 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v18.16b}, [%x[in]], #16\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" @@ -13489,37 +13470,37 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_256_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_256_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_256_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -13564,40 +13545,40 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v28.d[1], v31.d[0]\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "eor v26.16b, v28.16b, v30.16b\n\t" /* Done GHASH */ "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_256_out_start_byte_%=\n\t" @@ -13605,11 +13586,11 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_256_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "aese v14.16b, v0.16b\n\t" @@ -13645,19 +13626,19 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull2 v30.1q, v31.2d, v27.2d\n\t" - "ldr q11, [x9, #-32]\n\t" + "ldr q11, [%x[key], #-32]\n\t" "aese v14.16b, v10.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "mov v28.d[1], v31.d[0]\n\t" - "ldr q12, [x9, #-16]\n\t" + "ldr q12, [%x[key], #-16]\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v28.16b, v30.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "rbit v26.16b, v26.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v26.16b, v26.16b, v14.16b\n\t" @@ -13667,50 +13648,51 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_256_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_256_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_256_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_256_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -13731,7 +13713,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_128_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -13764,7 +13746,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13781,7 +13763,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -13798,7 +13780,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13815,7 +13797,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -13832,7 +13814,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13849,7 +13831,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -13866,7 +13848,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13884,7 +13866,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -13901,7 +13883,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -13922,7 +13904,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -13951,14 +13933,14 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -14000,7 +13982,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -14030,7 +14012,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -14059,7 +14041,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v26.16b, v31.16b\n\t" @@ -14087,7 +14069,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v26.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v30.16b, v30.16b, v26.16b\n\t" @@ -14115,7 +14097,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v30.16b, v30.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -14145,7 +14127,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -14174,7 +14156,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -14201,7 +14183,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v31.16b, v31.16b, v30.16b\n\t" @@ -14226,7 +14208,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -14248,7 +14230,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -14265,7 +14247,7 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -14377,10 +14359,10 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_128_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d}, [x9], #32\n\t" - "ld1 {v10.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d}, [%x[key]], #32\n\t" + "ld1 {v10.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_128_start_1_%=\n\t" @@ -14847,37 +14829,37 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_128_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_128_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_128_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -14922,30 +14904,30 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, /* Done GHASH */ "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_128_out_start_byte_%=\n\t" @@ -14953,11 +14935,11 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_128_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -15003,50 +14985,51 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_128_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_128_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_128_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_128_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -15060,12 +15043,11 @@ int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_gcm_decrypt_arm64_crypto_done_%=:\n\t" - "ldp x29, x30, [sp], #0x50\n\t" - : [out] "+r" (out), [sz] "+r" (sz), [nonceSz] "+r" (nonceSz), - [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), [key] "+r" (key), - [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), [reg] "+r" (reg), - [nr] "+r" (nr) - : [in] "r" (in), [nonce] "r" (nonce), [tag] "r" (tag), [aad] "r" (aad) + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), + [nonceSz] "+r" (nonceSz), [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), + [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), + [reg] "+r" (reg), [nr] "+r" (nr) + : [nonce] "r" (nonce), [tag] "r" (tag), [aad] "r" (aad) : "memory", "cc", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", @@ -15082,19 +15064,11 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-80]!\n\t" - "add x29, sp, #0\n\t" - "str %w[nr], [sp, #72]\n\t" - "str %x[reg], [sp, #64]\n\t" - "str %x[tmp], [sp, #56]\n\t" - "str %x[gcm_h], [sp, #48]\n\t" - "str %x[key], [sp, #40]\n\t" - "str %w[aadSz], [sp, #32]\n\t" "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" - "ld1 {v22.2d}, [x10]\n\t" - "cmp w8, #0x40\n\t" + "ld1 {v22.2d}, [%x[gcm_h]]\n\t" + "cmp %w[aadSz], #0x40\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #32\n\t" "csetm x17, lt\n\t" @@ -15109,7 +15083,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v31.1q, v29.2d, v27.2d\n\t" "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" - "cmp w8, #0x100\n\t" + "cmp %w[aadSz], #0x100\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x40\n\t" "csetm x17, lt\n\t" @@ -15139,7 +15113,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v30.d[1], v29.d[0]\n\t" "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ - "cmp w8, #0x400\n\t" + "cmp %w[aadSz], #0x400\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x200\n\t" "csetm x17, lt\n\t" @@ -15194,7 +15168,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_h_done_%=:\n\t" - "lsr w14, w8, #4\n\t" + "lsr w14, %w[aadSz], #4\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_1_%=\n\t" "cmp w14, #16\n\t" @@ -15417,41 +15391,41 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_both_1_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_aad_done_%=:\n\t" - "and w14, w8, #15\n\t" + "and w14, %w[aadSz], #15\n\t" "cbz w14, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w14\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_dw_%=\n\t" "ldr x19, [%x[aad]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_sw_%=\n\t" "ldr w19, [%x[aad]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte_%=\n\t" "ldrh w19, [%x[aad]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_encrypt_arm64_crypto_eor3_aad_end_bytes_%=\n\t" "ldrb w19, [%x[aad]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_aad_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_aad_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v26.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -15514,37 +15488,37 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "cbz x24, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w24\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_dw_%=\n\t" "ldr x19, [%x[nonce]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_sw_%=\n\t" "ldr w19, [%x[nonce]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte_%=\n\t" "ldrh w19, [%x[nonce]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes_%=\n\t" "ldrb w19, [%x[nonce]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_end_bytes_%=:\n\t" - "sub x11, x11, x24\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x24\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v13.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -15565,7 +15539,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_nonce_partial_done_%=:\n\t" "eor x14, x14, x14\n\t" - "lsl x24, %x[nonceSz], #3\n\t" + "ubfiz x24, %x[nonceSz], #3, #32\n\t" "mov v28.d[0], x14\n\t" "mov v28.d[1], x24\n\t" "rev64 v28.16b, v28.16b\n\t" @@ -15588,9 +15562,9 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "rev w15, w15\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_done_nonce_%=:\n\t" - "st1 {v13.2d}, [x12]\n\t" + "st1 {v13.2d}, [%x[reg]]\n\t" "lsr w14, %w[sz], #4\n\t" - "cmp w13, #12\n\t" + "cmp %w[nr], #12\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_start_128_%=\n\t" "b.gt L_aes_gcm_encrypt_arm64_crypto_eor3_start_256_%=\n\t" /* AES_GCM_192 */ @@ -15599,7 +15573,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -15632,7 +15606,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15649,7 +15623,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15666,7 +15640,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15683,7 +15657,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15700,7 +15674,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15717,7 +15691,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15734,7 +15708,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15752,7 +15726,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15769,7 +15743,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15790,7 +15764,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15811,7 +15785,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v3.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -15828,7 +15802,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -15853,14 +15827,14 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -15901,7 +15875,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -15931,7 +15905,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -15958,7 +15932,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -15986,7 +15960,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -16013,7 +15987,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -16039,7 +16013,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -16068,7 +16042,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -16095,7 +16069,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v31.16b, v31.16b, v29.16b, v30.16b\n\t" @@ -16119,7 +16093,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -16142,7 +16116,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -16159,7 +16133,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -16176,7 +16150,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -16193,7 +16167,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -16297,10 +16271,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_1_%=\n\t" @@ -16800,37 +16774,37 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -16860,31 +16834,31 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v14.16b, v11.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_192_out_start_byte_%=\n\t" @@ -16895,10 +16869,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_192_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -16918,11 +16892,11 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -16971,30 +16945,30 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_eor3_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_192_tag_start_byte_%=\n\t" @@ -17010,7 +16984,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -17043,7 +17017,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17060,7 +17034,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17077,7 +17051,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17094,7 +17068,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17111,7 +17085,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17128,7 +17102,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17145,7 +17119,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17163,7 +17137,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17180,7 +17154,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17201,7 +17175,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17222,7 +17196,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v3.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17239,7 +17213,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17256,7 +17230,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17273,7 +17247,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17298,14 +17272,14 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -17346,7 +17320,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -17376,7 +17350,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -17403,7 +17377,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -17431,7 +17405,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -17458,7 +17432,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17484,7 +17458,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -17513,7 +17487,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -17540,7 +17514,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v31.16b, v31.16b, v29.16b, v30.16b\n\t" @@ -17564,7 +17538,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -17587,7 +17561,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17604,7 +17578,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17621,7 +17595,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17638,7 +17612,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -17655,7 +17629,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -17672,7 +17646,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -17776,10 +17750,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9], #16\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]], #16\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_1_%=\n\t" @@ -17902,7 +17876,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18089,7 +18063,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18226,7 +18200,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v11.16b\n\t" "aesmc v15.16b, v15.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18297,10 +18271,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" @@ -18328,37 +18302,37 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -18387,38 +18361,38 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_256_out_start_byte_%=\n\t" @@ -18429,10 +18403,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_256_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -18452,11 +18426,11 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "aese v14.16b, v0.16b\n\t" @@ -18492,18 +18466,18 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "mov v28.d[1], v31.d[0]\n\t" - "ldr q11, [x9, #-32]\n\t" + "ldr q11, [%x[key], #-32]\n\t" "aese v14.16b, v10.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v28.16b, v30.16b\n\t" - "ldr q12, [x9, #-16]\n\t" + "ldr q12, [%x[key], #-16]\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "rbit v26.16b, v26.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v26.16b, v26.16b, v14.16b\n\t" @@ -18513,30 +18487,30 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_eor3_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_256_tag_start_byte_%=\n\t" @@ -18552,7 +18526,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -18585,7 +18559,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18602,7 +18576,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -18619,7 +18593,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18636,7 +18610,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -18653,7 +18627,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18670,7 +18644,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -18687,7 +18661,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18705,7 +18679,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -18722,7 +18696,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18743,7 +18717,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -18772,14 +18746,14 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v1.16b, v1.16b, v9.16b\n\t" "eor v2.16b, v2.16b, v10.16b\n\t" "eor v3.16b, v3.16b, v11.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v18.16b, v19.16b, v20.16b, v21.16b}, [%x[out]], #0x40\n\t" "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_8_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -18820,7 +18794,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v9.s[3], w19\n\t" "mov v10.s[3], w17\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" @@ -18850,7 +18824,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v2.16b, v2.16b, #8\n\t" @@ -18877,7 +18851,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v1.16b, v1.16b, #8\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v24.1d\n\t" @@ -18905,7 +18879,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v25.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -18932,7 +18906,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -18958,7 +18932,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^7 */ @@ -18987,7 +18961,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v7.2d, v18.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -19014,7 +18988,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v31.16b, v31.16b, v29.16b, v30.16b\n\t" @@ -19038,7 +19012,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v19.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v20.16b}, [%x[in]], #16\n\t" @@ -19061,7 +19035,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -19078,7 +19052,7 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v18.16b, v18.16b, v14.16b\n\t" "eor v19.16b, v19.16b, v15.16b\n\t" "eor v20.16b, v20.16b, v16.16b\n\t" @@ -19182,10 +19156,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d}, [x9], #32\n\t" - "ld1 {v10.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d}, [%x[key]], #32\n\t" + "ld1 {v10.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_done_%=\n\t" "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_1_%=\n\t" @@ -19641,37 +19615,37 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done_%=\n\t" "eor v16.16b, v16.16b, v16.16b\n\t" "mov w19, w14\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_byte_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v16.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v16.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rev w16, w15\n\t" @@ -19697,31 +19671,31 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "eor v14.16b, v14.16b, v10.16b\n\t" "eor v16.16b, v16.16b, v14.16b\n\t" - "st1 {v16.2d}, [x11]\n\t" + "st1 {v16.2d}, [%x[tmp]]\n\t" "mov w19, w14\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub x19, x19, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub x19, x19, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub x19, x19, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs x19, x19, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_128_out_start_byte_%=\n\t" @@ -19732,10 +19706,10 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero_%=:\n\t" "subs x17, x17, #1\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_128_start_zero_%=\n\t" - "sub x11, x11, #16\n\t" - "ld1 {v14.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], #16\n\t" + "ld1 {v14.2d}, [%x[tmp]]\n\t" "rbit v14.16b, v14.16b\n\t" "eor v15.16b, v26.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -19755,11 +19729,11 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -19804,30 +19778,30 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_encrypt_arm64_crypto_eor3_done_%=\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_partial_%=:\n\t" - "st1 {v26.16b}, [x11]\n\t" + "st1 {v26.16b}, [%x[tmp]]\n\t" "cmp %w[tagSz], #8\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_dw_%=\n\t" - "ldr x16, [x11], #8\n\t" + "ldr x16, [%x[tmp]], #8\n\t" "sub %w[tagSz], %w[tagSz], #8\n\t" "str x16, [%x[tag]], #8\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_dw_%=:\n\t" "cmp %w[tagSz], #4\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_sw_%=\n\t" - "ldr w16, [x11], #4\n\t" + "ldr w16, [%x[tmp]], #4\n\t" "sub %w[tagSz], %w[tagSz], #4\n\t" "str w16, [%x[tag]], #4\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_sw_%=:\n\t" "cmp %w[tagSz], #2\n\t" "b.lt L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte_%=\n\t" - "ldrh w16, [x11], #2\n\t" + "ldrh w16, [%x[tmp]], #2\n\t" "sub %w[tagSz], %w[tagSz], #2\n\t" "strh w16, [%x[tag]], #2\n\t" "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte_%=:\n\t" "cbz %w[tagSz], L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_end_bytes_%=\n\t" - "ldrb w16, [x11], #1\n\t" + "ldrb w16, [%x[tmp]], #1\n\t" "subs %w[tagSz], %w[tagSz], #1\n\t" "strb w16, [%x[tag]], #1\n\t" "b.ne L_aes_gcm_encrypt_arm64_crypto_eor3_128_tag_start_byte_%=\n\t" @@ -19836,7 +19810,6 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_gcm_encrypt_arm64_crypto_eor3_done_%=:\n\t" - "ldp x29, x30, [sp], #0x50\n\t" : [out] "+r" (out), [sz] "+r" (sz), [nonceSz] "+r" (nonceSz), [tag] "+r" (tag), [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), @@ -19857,19 +19830,11 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-80]!\n\t" - "add x29, sp, #0\n\t" - "str %w[nr], [sp, #72]\n\t" - "str %x[reg], [sp, #64]\n\t" - "str %x[tmp], [sp, #56]\n\t" - "str %x[gcm_h], [sp, #48]\n\t" - "str %x[key], [sp, #40]\n\t" - "str %w[aadSz], [sp, #32]\n\t" "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" - "ld1 {v22.2d}, [x10]\n\t" - "cmp w8, #0x40\n\t" + "ld1 {v22.2d}, [%x[gcm_h]]\n\t" + "cmp %w[aadSz], #0x40\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #32\n\t" "csetm x17, lt\n\t" @@ -19884,7 +19849,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v31.1q, v29.2d, v27.2d\n\t" "mov v30.d[1], v29.d[0]\n\t" "eor v23.16b, v30.16b, v31.16b\n\t" - "cmp w8, #0x100\n\t" + "cmp %w[aadSz], #0x100\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x40\n\t" "csetm x17, lt\n\t" @@ -19914,7 +19879,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v30.d[1], v29.d[0]\n\t" "eor v25.16b, v30.16b, v31.16b\n\t" /* Done */ - "cmp w8, #0x400\n\t" + "cmp %w[aadSz], #0x400\n\t" "csetm x16, lt\n\t" "cmp %w[sz], #0x200\n\t" "csetm x17, lt\n\t" @@ -19969,7 +19934,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done */ "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_h_done_%=:\n\t" - "lsr w14, w8, #4\n\t" + "lsr w14, %w[aadSz], #4\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_1_%=\n\t" "cmp w14, #16\n\t" @@ -20192,41 +20157,41 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_both_1_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_aad_done_%=:\n\t" - "and w14, w8, #15\n\t" + "and w14, %w[aadSz], #15\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w14\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_dw_%=\n\t" "ldr x19, [%x[aad]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_sw_%=\n\t" "ldr w19, [%x[aad]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte_%=\n\t" "ldrh w19, [%x[aad]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_decrypt_arm64_crypto_eor3_aad_end_bytes_%=\n\t" "ldrb w19, [%x[aad]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_aad_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_aad_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v26.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -20289,37 +20254,37 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "cbz x24, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done_%=\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov w20, w24\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp w20, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_dw_%=\n\t" "ldr x19, [%x[nonce]], #8\n\t" "sub w20, w20, #8\n\t" - "str x19, [x11], #8\n\t" + "str x19, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_dw_%=:\n\t" "cmp w20, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_sw_%=\n\t" "ldr w19, [%x[nonce]], #4\n\t" "sub w20, w20, #4\n\t" - "str w19, [x11], #4\n\t" + "str w19, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_sw_%=:\n\t" "cmp w20, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte_%=\n\t" "ldrh w19, [%x[nonce]], #2\n\t" "sub w20, w20, #2\n\t" - "strh w19, [x11], #2\n\t" + "strh w19, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte_%=:\n\t" "cbz w20, L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes_%=\n\t" "ldrb w19, [%x[nonce]], #1\n\t" "subs w20, w20, #1\n\t" - "strb w19, [x11], #1\n\t" + "strb w19, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_end_bytes_%=:\n\t" - "sub x11, x11, x24\n\t" - "ld1 {v18.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x24\n\t" + "ld1 {v18.2d}, [%x[tmp]]\n\t" "rbit v18.16b, v18.16b\n\t" "eor v21.16b, v13.16b, v18.16b\n\t" /* X = C * H^1 */ @@ -20340,7 +20305,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_nonce_partial_done_%=:\n\t" "eor x14, x14, x14\n\t" - "lsl x24, %x[nonceSz], #3\n\t" + "ubfiz x24, %x[nonceSz], #3, #32\n\t" "mov v28.d[0], x14\n\t" "mov v28.d[1], x24\n\t" "rev64 v28.16b, v28.16b\n\t" @@ -20363,9 +20328,9 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "rev w15, w15\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_done_nonce_%=:\n\t" - "st1 {v13.2d}, [x12]\n\t" + "st1 {v13.2d}, [%x[reg]]\n\t" "lsr w14, %w[sz], #4\n\t" - "cmp w13, #12\n\t" + "cmp %w[nr], #12\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_start_128_%=\n\t" "b.gt L_aes_gcm_decrypt_arm64_crypto_eor3_start_256_%=\n\t" /* AES_GCM_192 */ @@ -20374,7 +20339,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -20407,7 +20372,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20424,7 +20389,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20441,7 +20406,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20458,7 +20423,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20475,7 +20440,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20492,7 +20457,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20509,7 +20474,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20527,7 +20492,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20544,7 +20509,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20565,7 +20530,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20586,7 +20551,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20603,7 +20568,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20628,14 +20593,14 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -20677,7 +20642,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -20707,7 +20672,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -20735,7 +20700,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -20762,7 +20727,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20788,7 +20753,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -20817,7 +20782,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -20845,7 +20810,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -20871,7 +20836,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull2 v30.1q, v31.2d, v27.2d\n\t" @@ -20895,7 +20860,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -20917,7 +20882,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20934,7 +20899,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -20951,7 +20916,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -20968,7 +20933,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -21072,10 +21037,10 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_1_%=\n\t" @@ -21576,37 +21541,37 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_192_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -21654,30 +21619,30 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_start_byte_%=\n\t" @@ -21685,11 +21650,11 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_eor3_192_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -21738,50 +21703,51 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_192_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_192_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -21802,7 +21768,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -21835,7 +21801,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -21852,7 +21818,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -21869,7 +21835,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -21886,7 +21852,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -21903,7 +21869,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -21920,7 +21886,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -21937,7 +21903,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -21955,7 +21921,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -21972,7 +21938,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -21993,7 +21959,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22014,7 +21980,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -22031,7 +21997,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22048,7 +22014,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -22065,7 +22031,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22090,14 +22056,14 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -22139,7 +22105,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -22169,7 +22135,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -22197,7 +22163,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -22224,7 +22190,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22250,7 +22216,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -22279,7 +22245,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -22307,7 +22273,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -22333,7 +22299,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull2 v30.1q, v31.2d, v27.2d\n\t" @@ -22357,7 +22323,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -22379,7 +22345,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22396,7 +22362,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #176]\n\t" + "ldr q13, [%x[key], #176]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -22413,7 +22379,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #192]\n\t" + "ldr q12, [%x[key], #192]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22430,7 +22396,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #208]\n\t" + "ldr q13, [%x[key], #208]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -22447,7 +22413,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #224]\n\t" + "ldr q12, [%x[key], #224]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -22464,7 +22430,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -22568,10 +22534,10 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [x9], #0x40\n\t" - "ld1 {v12.2d}, [x9], #16\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d, v10.2d, v11.2d}, [%x[key]], #0x40\n\t" + "ld1 {v12.2d}, [%x[key]], #16\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_1_%=\n\t" @@ -22694,7 +22660,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -22881,7 +22847,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v16.16b, v16.16b\n\t" "aese v17.16b, v11.16b\n\t" "aesmc v17.16b, v17.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23018,7 +22984,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v11.16b\n\t" "aesmc v15.16b, v15.16b\n\t" - "ld1 {v29.2d, v30.2d}, [x9]\n\t" + "ld1 {v29.2d, v30.2d}, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23089,10 +23055,10 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v18.16b}, [%x[in]], #16\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" @@ -23120,37 +23086,37 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_256_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -23195,39 +23161,39 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v26.16b, v28.16b, v30.16b\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" /* Done GHASH */ "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_start_byte_%=\n\t" @@ -23235,11 +23201,11 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_eor3_256_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "aese v14.16b, v0.16b\n\t" @@ -23275,18 +23241,18 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v14.16b, v9.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "mov v28.d[1], v31.d[0]\n\t" - "ldr q11, [x9, #-32]\n\t" + "ldr q11, [%x[key], #-32]\n\t" "aese v14.16b, v10.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v26.16b, v28.16b, v30.16b\n\t" - "ldr q12, [x9, #-16]\n\t" + "ldr q12, [%x[key], #-16]\n\t" "aese v14.16b, v11.16b\n\t" "aesmc v14.16b, v14.16b\n\t" - "ldr q29, [x9]\n\t" + "ldr q29, [%x[key]]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "rbit v26.16b, v26.16b\n\t" - "ldr q30, [x9, #16]\n\t" + "ldr q30, [%x[key], #16]\n\t" "aese v14.16b, v29.16b\n\t" "eor v14.16b, v14.16b, v30.16b\n\t" "eor v26.16b, v26.16b, v14.16b\n\t" @@ -23296,50 +23262,51 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_256_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_256_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -23360,7 +23327,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -23393,7 +23360,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "mov v10.s[3], w17\n\t" "rev w16, w15\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23410,7 +23377,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23427,7 +23394,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23444,7 +23411,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23461,7 +23428,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23478,7 +23445,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23495,7 +23462,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23513,7 +23480,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23530,7 +23497,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v12.16b\n\t" @@ -23551,7 +23518,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23580,14 +23547,14 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v9.16b, v9.16b, v1.16b\n\t" "eor v10.16b, v10.16b, v2.16b\n\t" "eor v11.16b, v11.16b, v3.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "st1 {v14.16b, v15.16b, v16.16b, v17.16b}, [%x[out]], #0x40\n\t" "st1 {v8.16b, v9.16b, v10.16b, v11.16b}, [%x[out]], #0x40\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_8_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_both_8_%=:\n\t" - "ldr q12, [x9]\n\t" + "ldr q12, [%x[key]]\n\t" "add w24, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "add w23, w15, #2\n\t" @@ -23629,7 +23596,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "rev w16, w15\n\t" "eor v18.16b, v18.16b, v26.16b\n\t" "mov v11.s[3], w16\n\t" - "ldr q13, [x9, #16]\n\t" + "ldr q13, [%x[key], #16]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X = C * H^1 */ @@ -23659,7 +23626,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ext v26.16b, v2.16b, v2.16b, #8\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #32]\n\t" + "ldr q12, [%x[key], #32]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull v31.1q, v26.1d, v23.1d\n\t" @@ -23687,7 +23654,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v26.2d, v24.2d\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #48]\n\t" + "ldr q13, [%x[key], #48]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" @@ -23714,7 +23681,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor3 v30.16b, v30.16b, v26.16b, v31.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #64]\n\t" + "ldr q12, [%x[key], #64]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23740,7 +23707,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #80]\n\t" + "ldr q13, [%x[key], #80]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" /* X += C * H^6 */ @@ -23769,7 +23736,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "pmull2 v26.1q, v6.2d, v19.2d\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #96]\n\t" + "ldr q12, [%x[key], #96]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "eor v28.16b, v28.16b, v31.16b\n\t" @@ -23797,7 +23764,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v29.16b, v29.16b, v26.16b\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #112]\n\t" + "ldr q13, [%x[key], #112]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ext v26.16b, v18.16b, v18.16b, #8\n\t" @@ -23823,7 +23790,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" "subs w14, w14, #8\n\t" - "ldr q12, [x9, #128]\n\t" + "ldr q12, [%x[key], #128]\n\t" "aese v14.16b, v13.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "pmull2 v30.1q, v31.2d, v27.2d\n\t" @@ -23847,7 +23814,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "ld1 {v20.16b}, [%x[in]], #16\n\t" "aese v11.16b, v13.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q13, [x9, #144]\n\t" + "ldr q13, [%x[key], #144]\n\t" "aese v14.16b, v12.16b\n\t" "aesmc v14.16b, v14.16b\n\t" "ld1 {v21.16b}, [%x[in]], #16\n\t" @@ -23869,7 +23836,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "aesmc v10.16b, v10.16b\n\t" "aese v11.16b, v12.16b\n\t" "aesmc v11.16b, v11.16b\n\t" - "ldr q12, [x9, #160]\n\t" + "ldr q12, [%x[key], #160]\n\t" "aese v14.16b, v13.16b\n\t" "eor v14.16b, v14.16b, v12.16b\n\t" "aese v15.16b, v13.16b\n\t" @@ -23886,7 +23853,7 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "eor v10.16b, v10.16b, v12.16b\n\t" "aese v11.16b, v13.16b\n\t" "eor v11.16b, v11.16b, v12.16b\n\t" - "ld1 {v13.2d}, [x12]\n\t" + "ld1 {v13.2d}, [%x[reg]]\n\t" "eor v14.16b, v14.16b, v18.16b\n\t" "eor v15.16b, v15.16b, v19.16b\n\t" "eor v16.16b, v16.16b, v20.16b\n\t" @@ -23990,10 +23957,10 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_4_%=:\n\t" - "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [x9], #0x40\n\t" - "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [x9], #0x40\n\t" - "ld1 {v8.2d, v9.2d}, [x9], #32\n\t" - "ld1 {v10.2d}, [x9]\n\t" + "ld1 {v0.2d, v1.2d, v2.2d, v3.2d}, [%x[key]], #0x40\n\t" + "ld1 {v4.2d, v5.2d, v6.2d, v7.2d}, [%x[key]], #0x40\n\t" + "ld1 {v8.2d, v9.2d}, [%x[key]], #32\n\t" + "ld1 {v10.2d}, [%x[key]]\n\t" "cmp w14, #1\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_done_%=\n\t" "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_1_%=\n\t" @@ -24449,37 +24416,37 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b.eq L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done_%=\n\t" "eor v15.16b, v15.16b, v15.16b\n\t" "mov w19, w14\n\t" - "st1 {v15.2d}, [x11]\n\t" + "st1 {v15.2d}, [%x[tmp]]\n\t" "cmp x19, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_dw_%=\n\t" "ldr x17, [%x[in]], #8\n\t" "sub x19, x19, #8\n\t" - "str x17, [x11], #8\n\t" + "str x17, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_dw_%=:\n\t" "cmp x19, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_sw_%=\n\t" "ldr w17, [%x[in]], #4\n\t" "sub x19, x19, #4\n\t" - "str w17, [x11], #4\n\t" + "str w17, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_sw_%=:\n\t" "cmp x19, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte_%=\n\t" "ldrh w17, [%x[in]], #2\n\t" "sub x19, x19, #2\n\t" - "strh w17, [x11], #2\n\t" + "strh w17, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte_%=:\n\t" "cbz x19, L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_bytes_%=\n\t" "ldrb w17, [%x[in]], #1\n\t" "subs x19, x19, #1\n\t" - "strb w17, [x11], #1\n\t" + "strb w17, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_128_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_end_bytes_%=:\n\t" - "sub x11, x11, x14\n\t" - "ld1 {v15.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], x14\n\t" + "ld1 {v15.2d}, [%x[tmp]]\n\t" "add w15, w15, #1\n\t" "mov v14.16b, v13.16b\n\t" "rbit v15.16b, v15.16b\n\t" @@ -24523,30 +24490,30 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, /* Done GHASH */ "rbit v15.16b, v15.16b\n\t" "eor v14.16b, v14.16b, v15.16b\n\t" - "st1 {v14.2d}, [x11]\n\t" + "st1 {v14.2d}, [%x[tmp]]\n\t" "cmp w14, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_dw_%=\n\t" - "ldr x17, [x11], #8\n\t" + "ldr x17, [%x[tmp]], #8\n\t" "sub w14, w14, #8\n\t" "str x17, [%x[out]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_dw_%=:\n\t" "cmp w14, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_sw_%=\n\t" - "ldr w17, [x11], #4\n\t" + "ldr w17, [%x[tmp]], #4\n\t" "sub w14, w14, #4\n\t" "str w17, [%x[out]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_sw_%=:\n\t" "cmp w14, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte_%=\n\t" - "ldrh w17, [x11], #2\n\t" + "ldrh w17, [%x[tmp]], #2\n\t" "sub w14, w14, #2\n\t" "strh w17, [%x[out]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte_%=:\n\t" "cbz w14, L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes_%=\n\t" - "ldrb w17, [x11], #1\n\t" + "ldrb w17, [%x[tmp]], #1\n\t" "subs w14, w14, #1\n\t" "strb w17, [%x[out]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_start_byte_%=\n\t" @@ -24554,11 +24521,11 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "L_aes_gcm_decrypt_arm64_crypto_eor3_128_out_end_bytes_%=:\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_partial_done_%=:\n\t" - "ld1 {v14.2d}, [x12]\n\t" - "lsl x8, x8, #3\n\t" - "rbit x8, x8\n\t" - "mov v28.d[0], x8\n\t" - "lsl %x[sz], %x[sz], #3\n\t" + "ld1 {v14.2d}, [%x[reg]]\n\t" + "ubfiz %x[aadSz], %x[aadSz], #3, #32\n\t" + "rbit %x[aadSz], %x[aadSz]\n\t" + "mov v28.d[0], %x[aadSz]\n\t" + "ubfiz %x[sz], %x[sz], #3, #32\n\t" "rbit %x[sz], %x[sz]\n\t" "mov v28.d[1], %x[sz]\n\t" "eor v26.16b, v26.16b, v28.16b\n\t" @@ -24603,50 +24570,51 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, "b L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_part_tag_%=:\n\t" + "ubfiz %x[tagSz], %x[tagSz], #0, #32\n\t" "eor v28.16b, v28.16b, v28.16b\n\t" "mov x17, %x[tagSz]\n\t" - "st1 {v28.2d}, [x11]\n\t" + "st1 {v28.2d}, [%x[tmp]]\n\t" "cmp x17, #8\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_dw_%=\n\t" "ldr x16, [%x[tag]], #8\n\t" "sub x17, x17, #8\n\t" - "str x16, [x11], #8\n\t" + "str x16, [%x[tmp]], #8\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_dw_%=:\n\t" "cmp x17, #4\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_sw_%=\n\t" "ldr w16, [%x[tag]], #4\n\t" "sub x17, x17, #4\n\t" - "str w16, [x11], #4\n\t" + "str w16, [%x[tmp]], #4\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_sw_%=:\n\t" "cmp x17, #2\n\t" "b.lt L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte_%=\n\t" "ldrh w16, [%x[tag]], #2\n\t" "sub x17, x17, #2\n\t" - "strh w16, [x11], #2\n\t" + "strh w16, [%x[tmp]], #2\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte_%=:\n\t" "cbz x17, L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_end_bytes_%=\n\t" "ldrb w16, [%x[tag]], #1\n\t" "subs x17, x17, #1\n\t" - "strb w16, [x11], #1\n\t" + "strb w16, [%x[tmp]], #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_start_byte_%=\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_end_bytes_%=:\n\t" - "sub x11, x11, %x[tagSz]\n\t" - "ld1 {v28.2d}, [x11]\n\t" + "sub %x[tmp], %x[tmp], %x[tagSz]\n\t" + "ld1 {v28.2d}, [%x[tmp]]\n\t" "mov x17, #16\n\t" - "st1 {v26.2d}, [x11]\n\t" + "st1 {v26.2d}, [%x[tmp]]\n\t" "sub x17, x17, %x[tagSz]\n\t" - "add x11, x11, %x[tagSz]\n\t" + "add %x[tmp], %x[tmp], %x[tagSz]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte_%=:\n\t" - "strb wzr, [x11], #1\n\t" + "strb wzr, [%x[tmp]], #1\n\t" "subs x17, x17, #1\n\t" "b.ne L_aes_gcm_decrypt_arm64_crypto_eor3_128_calc_tag_byte_%=\n\t" - "subs x11, x11, #16\n\t" - "ld1 {v26.2d}, [x11]\n\t" + "subs %x[tmp], %x[tmp], #16\n\t" + "ld1 {v26.2d}, [%x[tmp]]\n\t" "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_128_tag_loaded_%=:\n\t" "eor v28.16b, v28.16b, v26.16b\n\t" @@ -24660,12 +24628,11 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_gcm_decrypt_arm64_crypto_eor3_done_%=:\n\t" - "ldp x29, x30, [sp], #0x50\n\t" - : [out] "+r" (out), [sz] "+r" (sz), [nonceSz] "+r" (nonceSz), - [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), [key] "+r" (key), - [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), [reg] "+r" (reg), - [nr] "+r" (nr) - : [in] "r" (in), [nonce] "r" (nonce), [tag] "r" (tag), [aad] "r" (aad) + : [in] "+r" (in), [out] "+r" (out), [sz] "+r" (sz), + [nonceSz] "+r" (nonceSz), [tagSz] "+r" (tagSz), [aadSz] "+r" (aadSz), + [key] "+r" (key), [gcm_h] "+r" (gcm_h), [tmp] "+r" (tmp), + [reg] "+r" (reg), [nr] "+r" (nr) + : [nonce] "r" (nonce), [tag] "r" (tag), [aad] "r" (aad) : "memory", "cc", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", @@ -24781,7 +24748,7 @@ void AES_GCM_init_AARCH64(byte* key, int nr, const byte* nonce, word32 nonceSz, "\n" "L_aes_gcm_init_arm64_crypto_partial_done_%=:\n\t" "eor x7, x7, x7\n\t" - "lsl x13, %x[nonceSz], #3\n\t" + "ubfiz x13, %x[nonceSz], #3, #32\n\t" "mov v7.d[0], x7\n\t" "mov v7.d[1], x13\n\t" "rev64 v7.16b, v7.16b\n\t" @@ -25299,8 +25266,6 @@ void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -29113,7 +29078,6 @@ void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov v13.s[3], w9\n\t" "st1 {v26.2d}, [%x[tag]]\n\t" "st1 {v13.2d}, [%x[counter]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [nr] "+r" (nr), [out] "+r" (out), [nbytes] "+r" (nbytes), [tag] "+r" (tag), [h] "+r" (h), [counter] "+r" (counter) : [key] "r" (key), [in] "r" (in) @@ -29134,10 +29098,10 @@ void AES_GCM_encrypt_final_AARCH64(byte* tag, byte* authTag, word32 tbytes, "ld1 {v4.2d}, [%x[h]]\n\t" "ushr v6.2d, v6.2d, #56\n\t" "ld1 {v7.2d}, [%x[initCtr]]\n\t" - "lsl %x[abytes], %x[abytes], #3\n\t" + "ubfiz %x[abytes], %x[abytes], #3, #32\n\t" "rbit %x[abytes], %x[abytes]\n\t" "mov v0.d[0], %x[abytes]\n\t" - "lsl %x[nbytes], %x[nbytes], #3\n\t" + "ubfiz %x[nbytes], %x[nbytes], #3, #32\n\t" "rbit %x[nbytes], %x[nbytes]\n\t" "mov v0.d[1], %x[nbytes]\n\t" "eor v5.16b, v5.16b, v0.16b\n\t" @@ -29207,8 +29171,6 @@ void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -33022,7 +32984,6 @@ void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, "mov v13.s[3], w9\n\t" "st1 {v26.2d}, [%x[tag]]\n\t" "st1 {v13.2d}, [%x[counter]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [nr] "+r" (nr), [out] "+r" (out), [nbytes] "+r" (nbytes), [tag] "+r" (tag), [h] "+r" (h), [counter] "+r" (counter) : [key] "r" (key), [in] "r" (in) @@ -33039,17 +33000,15 @@ void AES_GCM_decrypt_final_AARCH64(byte* tag, const byte* authTag, int* res) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v5.2d}, [%x[tag]]\n\t" "movi v6.16b, #0x87\n\t" "ld1 {v4.2d}, [%x[h]]\n\t" "ushr v6.2d, v6.2d, #56\n\t" "ld1 {v7.2d}, [%x[initCtr]]\n\t" - "lsl %x[abytes], %x[abytes], #3\n\t" + "ubfiz %x[abytes], %x[abytes], #3, #32\n\t" "rbit %x[abytes], %x[abytes]\n\t" "mov v0.d[0], %x[abytes]\n\t" - "lsl %x[nbytes], %x[nbytes], #3\n\t" + "ubfiz %x[nbytes], %x[nbytes], #3, #32\n\t" "rbit %x[nbytes], %x[nbytes]\n\t" "mov v0.d[1], %x[nbytes]\n\t" "eor v5.16b, v5.16b, v0.16b\n\t" @@ -33075,6 +33034,7 @@ void AES_GCM_decrypt_final_AARCH64(byte* tag, const byte* authTag, "b L_aes_gcm_decrypt_final_arm64_crypto_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_final_arm64_crypto_part_tag_%=:\n\t" + "ubfiz %x[tbytes], %x[tbytes], #0, #32\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x10, %x[tbytes]\n\t" "st1 {v0.2d}, [%x[tag]]\n\t" @@ -33131,7 +33091,6 @@ void AES_GCM_decrypt_final_AARCH64(byte* tag, const byte* authTag, "and x8, x8, x11\n\t" "add w8, w8, #0xb4\n\t" "str w8, [%x[res]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [tag] "+r" (tag), [tbytes] "+r" (tbytes), [nbytes] "+r" (nbytes), [abytes] "+r" (abytes), [h] "+r" (h), [initCtr] "+r" (initCtr), [res] "+r" (res) @@ -33243,7 +33202,7 @@ void AES_GCM_init_AARCH64_EOR3(byte* key, int nr, const byte* nonce, "\n" "L_aes_gcm_init_arm64_crypto_eor3_partial_done_%=:\n\t" "eor x7, x7, x7\n\t" - "lsl x13, %x[nonceSz], #3\n\t" + "ubfiz x13, %x[nonceSz], #3, #32\n\t" "mov v7.d[0], x7\n\t" "mov v7.d[1], x13\n\t" "rev64 v7.16b, v7.16b\n\t" @@ -33741,8 +33700,6 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -37471,7 +37428,6 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov v13.s[3], w9\n\t" "st1 {v26.2d}, [%x[tag]]\n\t" "st1 {v13.2d}, [%x[counter]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [nr] "+r" (nr), [out] "+r" (out), [nbytes] "+r" (nbytes), [tag] "+r" (tag), [h] "+r" (h), [counter] "+r" (counter) : [key] "r" (key), [in] "r" (in) @@ -37492,10 +37448,10 @@ void AES_GCM_encrypt_final_AARCH64_EOR3(byte* tag, byte* authTag, word32 tbytes, "ld1 {v4.2d}, [%x[h]]\n\t" "ushr v6.2d, v6.2d, #56\n\t" "ld1 {v7.2d}, [%x[initCtr]]\n\t" - "lsl %x[abytes], %x[abytes], #3\n\t" + "ubfiz %x[abytes], %x[abytes], #3, #32\n\t" "rbit %x[abytes], %x[abytes]\n\t" "mov v0.d[0], %x[abytes]\n\t" - "lsl %x[nbytes], %x[nbytes], #3\n\t" + "ubfiz %x[nbytes], %x[nbytes], #3, #32\n\t" "rbit %x[nbytes], %x[nbytes]\n\t" "mov v0.d[1], %x[nbytes]\n\t" "eor v5.16b, v5.16b, v0.16b\n\t" @@ -37564,8 +37520,6 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -41295,7 +41249,6 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, "mov v13.s[3], w9\n\t" "st1 {v26.2d}, [%x[tag]]\n\t" "st1 {v13.2d}, [%x[counter]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [nr] "+r" (nr), [out] "+r" (out), [nbytes] "+r" (nbytes), [tag] "+r" (tag), [h] "+r" (h), [counter] "+r" (counter) : [key] "r" (key), [in] "r" (in) @@ -41312,17 +41265,15 @@ void AES_GCM_decrypt_final_AARCH64_EOR3(byte* tag, const byte* authTag, int* res) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v5.2d}, [%x[tag]]\n\t" "movi v6.16b, #0x87\n\t" "ld1 {v4.2d}, [%x[h]]\n\t" "ushr v6.2d, v6.2d, #56\n\t" "ld1 {v7.2d}, [%x[initCtr]]\n\t" - "lsl %x[abytes], %x[abytes], #3\n\t" + "ubfiz %x[abytes], %x[abytes], #3, #32\n\t" "rbit %x[abytes], %x[abytes]\n\t" "mov v0.d[0], %x[abytes]\n\t" - "lsl %x[nbytes], %x[nbytes], #3\n\t" + "ubfiz %x[nbytes], %x[nbytes], #3, #32\n\t" "rbit %x[nbytes], %x[nbytes]\n\t" "mov v0.d[1], %x[nbytes]\n\t" "eor v5.16b, v5.16b, v0.16b\n\t" @@ -41347,6 +41298,7 @@ void AES_GCM_decrypt_final_AARCH64_EOR3(byte* tag, const byte* authTag, "b L_aes_gcm_decrypt_final_arm64_crypto_eor3_tag_loaded_%=\n\t" "\n" "L_aes_gcm_decrypt_final_arm64_crypto_eor3_part_tag_%=:\n\t" + "ubfiz %x[tbytes], %x[tbytes], #0, #32\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x10, %x[tbytes]\n\t" "st1 {v0.2d}, [%x[tag]]\n\t" @@ -41403,7 +41355,6 @@ void AES_GCM_decrypt_final_AARCH64_EOR3(byte* tag, const byte* authTag, "and x8, x8, x11\n\t" "add w8, w8, #0xb4\n\t" "str w8, [%x[res]]\n\t" - "ldp x29, x30, [sp], #32\n\t" : [tag] "+r" (tag), [tbytes] "+r" (tbytes), [nbytes] "+r" (nbytes), [abytes] "+r" (abytes), [h] "+r" (h), [initCtr] "+r" (initCtr), [res] "+r" (res) @@ -41421,8 +41372,6 @@ void AES_XTS_encrypt_AARCH64(const byte* in, byte* out, word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key2]], #0x40\n\t" "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key2]], #0x40\n\t" "ld1 {v4.16b}, [%x[i]]\n\t" @@ -42432,7 +42381,6 @@ void AES_XTS_encrypt_AARCH64(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_xts_encrypt_arm64_crypto_done_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i) @@ -42448,8 +42396,6 @@ void AES_XTS_decrypt_AARCH64(const byte* in, byte* out, word32 sz, const byte* i, byte* key, byte* key2, byte* tmp, int nr) { __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v16.2d, v17.2d, v18.2d, v19.2d}, [%x[key2]], #0x40\n\t" "ld1 {v20.2d, v21.2d, v22.2d, v23.2d}, [%x[key2]], #0x40\n\t" "ld1 {v4.16b}, [%x[i]]\n\t" @@ -43554,7 +43500,6 @@ void AES_XTS_decrypt_AARCH64(const byte* in, byte* out, word32 sz, #endif /* !NO_AES_128 */ "\n" "L_aes_xts_decrypt_arm64_crypto_done_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i) @@ -48897,8 +48842,6 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, const word8* te = L_AES_ARM64_NEON_te; const word8* shuffle = L_AES_ARM64_NEON_shift_rows_shuffle; __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [%[te]], #0x40\n\t" "ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [%[te]], #0x40\n\t" "ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [%[te]], #0x40\n\t" @@ -49944,7 +49887,6 @@ void AES_XTS_encrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "st1 {v0.16b}, [%x[out]]\n\t" "\n" "L_AES_XTS_encrypt_NEON_data_done_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i), [te] "r" (te), [shuffle] "r" (shuffle) @@ -49965,8 +49907,6 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, const word8* shuffle = L_AES_ARM64_NEON_shift_rows_shuffle; const word8* invshuffle = L_AES_ARM64_NEON_shift_rows_invshuffle; __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [%[te]], #0x40\n\t" "ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [%[te]], #0x40\n\t" "ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [%[te]], #0x40\n\t" @@ -51475,7 +51415,6 @@ void AES_XTS_decrypt_NEON(const byte* in, byte* out, word32 sz, const byte* i, "st1 {v0.16b}, [%x[out]]\n\t" "\n" "L_AES_XTS_decrypt_NEON_data_done_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i), [te] "r" (te), [td] "r" (td), @@ -54660,8 +54599,6 @@ void AES_XTS_encrypt(const byte* in, byte* out, word32 sz, const byte* i, { const word32* te = L_AES_ARM64_te; __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "mov x9, #0x87\n\t" "mov x26, %x[key2]\n\t" "ldp x21, x22, [%x[i]]\n\t" @@ -55586,7 +55523,6 @@ void AES_XTS_encrypt(const byte* in, byte* out, word32 sz, const byte* i, "stp x10, x11, [%x[out]]\n\t" "\n" "L_AES_XTS_encrypt_done_data_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i), [te] "r" (te) @@ -55603,8 +55539,6 @@ void AES_XTS_decrypt(const byte* in, byte* out, word32 sz, const byte* i, const word8* td4 = L_AES_ARM64_td4; const word32* te = L_AES_ARM64_te; __asm__ __volatile__ ( - "stp x29, x30, [sp, #-32]!\n\t" - "add x29, sp, #0\n\t" "ands w11, %w[sz], #15\n\t" "cset w11, ne\n\t" "sub %w[sz], %w[sz], w11, lsl 4\n\t" @@ -56753,7 +56687,6 @@ void AES_XTS_decrypt(const byte* in, byte* out, word32 sz, const byte* i, "stp x12, x13, [%x[out]]\n\t" "\n" "L_AES_XTS_decrypt_done_data_%=:\n\t" - "ldp x29, x30, [sp], #32\n\t" : [out] "+r" (out), [sz] "+r" (sz), [key] "+r" (key), [key2] "+r" (key2), [tmp] "+r" (tmp), [nr] "+r" (nr) : [in] "r" (in), [i] "r" (i), [td] "r" (td), [td4] "r" (td4), diff --git a/wolfcrypt/src/port/arm/armv8-curve25519.S b/wolfcrypt/src/port/arm/armv8-curve25519.S index 6f88bd5e19..539672529d 100644 --- a/wolfcrypt/src/port/arm/armv8-curve25519.S +++ b/wolfcrypt/src/port/arm/armv8-curve25519.S @@ -5401,6 +5401,14 @@ L_curve25519_bits: adcs x7, x7, x25 adcs x8, x8, x26 adc x9, x9, x27 + # Reduce if top bit set + mov x3, #19 + and x4, x3, x9, asr 63 + adds x6, x6, x4 + adcs x7, x7, xzr + and x9, x9, #0x7fffffffffffffff + adcs x8, x8, xzr + adc x9, x9, xzr # Square # A[0] * A[1] umulh x16, x19, x20 diff --git a/wolfcrypt/src/port/arm/armv8-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-curve25519_c.c index b8515b0054..9f3cecfc81 100644 --- a/wolfcrypt/src/port/arm/armv8-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-curve25519_c.c @@ -38,7 +38,7 @@ #if !defined(CURVE25519_SMALL) || !defined(ED25519_SMALL) #include -void fe_init() +void fe_init(void) { __asm__ __volatile__ ( "\n\t" @@ -229,8 +229,8 @@ int fe_isnonzero(const fe a) "orr %x[a], x1, x2\n\t" "orr x3, x3, x4\n\t" "orr %x[a], %x[a], x3\n\t" + : [a] "+r" (a) : - : [a] "r" (a) : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6" ); return (word32)(size_t)a; @@ -248,8 +248,8 @@ int fe_isnegative(const fe a) "adc x5, x4, xzr\n\t" "and %x[a], x1, #1\n\t" "eor %x[a], %x[a], x5, lsr 63\n\t" + : [a] "+r" (a) : - : [a] "r" (a) : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6" ); return (word32)(size_t)a; @@ -4362,7 +4362,7 @@ int curve25519_base(byte* r, const byte* n) /* Store */ "stp x14, x15, [%x[r]]\n\t" "stp x16, x17, [%x[r], #16]\n\t" - "mov x0, xzr\n\t" + "mov %x[r], xzr\n\t" "ldp x29, x30, [sp], #0xb0\n\t" : [r] "+r" (r) : [n] "r" (n), [x2] "r" (x2) @@ -5271,6 +5271,14 @@ int curve25519(byte* r, const byte* n, const byte* a) "adcs x7, x7, x25\n\t" "adcs x8, x8, x26\n\t" "adc x9, x9, x27\n\t" + /* Reduce if top bit set */ + "mov x3, #19\n\t" + "and x4, x3, x9, asr 63\n\t" + "adds x6, x6, x4\n\t" + "adcs x7, x7, xzr\n\t" + "and x9, x9, #0x7fffffffffffffff\n\t" + "adcs x8, x8, xzr\n\t" + "adc x9, x9, xzr\n\t" /* Square */ /* A[0] * A[1] */ "umulh x16, x19, x20\n\t" @@ -6969,7 +6977,7 @@ int curve25519(byte* r, const byte* n, const byte* a) /* Store */ "stp x14, x15, [%x[r]]\n\t" "stp x16, x17, [%x[r], #16]\n\t" - "mov x0, xzr\n\t" + "mov %x[r], xzr\n\t" "ldp x29, x30, [sp], #0xc0\n\t" : [r] "+r" (r) : [n] "r" (n), [a] "r" (a) diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c index 54a92c47fc..0c00ffbb48 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c @@ -8406,11 +8406,11 @@ int mlkem_cmp_neon(const byte* a, const byte* b, int sz) "orr v8.16b, v8.16b, v10.16b\n\t" "ext v9.16b, v8.16b, v8.16b, #8\n\t" "orr v8.16b, v8.16b, v9.16b\n\t" - "mov x0, v8.d[0]\n\t" - "subs x0, x0, xzr\n\t" - "csetm w0, ne\n\t" - : [sz] "+r" (sz) - : [a] "r" (a), [b] "r" (b) + "mov %x[a], v8.d[0]\n\t" + "subs %x[a], %x[a], xzr\n\t" + "csetm %w[a], ne\n\t" + : [a] "+r" (a), [sz] "+r" (sz) + : [b] "r" (b) : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11" ); @@ -9089,7 +9089,7 @@ unsigned int mlkem_rej_uniform_neon(sword16* p, unsigned int len, const byte* r, "b L_mlkem_rej_uniform_loop_lt_4_%=\n\t" "\n" "L_mlkem_rej_uniform_done_%=:\n\t" - "mov x0, x12\n\t" + "mov %x[p], x12\n\t" : [p] "+r" (p), [len] "+r" (len), [rLen] "+r" (rLen) : [r] "r" (r), [mask] "r" (mask), [q] "r" (q), [bits] "r" (bits), [indices] "r" (indices) diff --git a/wolfcrypt/src/port/arm/thumb2-curve25519.S b/wolfcrypt/src/port/arm/thumb2-curve25519.S index 29d3399a0c..621a20e5f1 100644 --- a/wolfcrypt/src/port/arm/thumb2-curve25519.S +++ b/wolfcrypt/src/port/arm/thumb2-curve25519.S @@ -1887,6 +1887,403 @@ fe_mul_op: POP {pc} /* Cycle Count = 406 */ .size fe_mul_op,.-fe_mul_op + .text + .align 4 + .globl fe_mul_op_full_red + .type fe_mul_op_full_red, %function +fe_mul_op_full_red: + PUSH {lr} + SUB sp, sp, #40 + STR r0, [sp, #36] + MOV r0, #0 + LDR r12, [r1] + /* A[0] * B[0] */ + LDR lr, [r2] + UMULL r3, r4, r12, lr + /* A[0] * B[2] */ + LDR lr, [r2, #8] + UMULL r5, r6, r12, lr + /* A[0] * B[4] */ + LDR lr, [r2, #16] + UMULL r7, r8, r12, lr + /* A[0] * B[6] */ + LDR lr, [r2, #24] + UMULL r9, r10, r12, lr + STR r3, [sp] + /* A[0] * B[1] */ + LDR lr, [r2, #4] + MOV r11, r0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[0] * B[3] */ + LDR lr, [r2, #12] + ADCS r6, r6, #0 + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[0] * B[5] */ + LDR lr, [r2, #20] + ADCS r8, r8, #0 + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[0] * B[7] */ + LDR lr, [r2, #28] + ADCS r10, r10, #0 + ADC r3, r0, #0 + UMLAL r10, r3, r12, lr + /* A[1] * B[0] */ + LDR r12, [r1, #4] + LDR lr, [r2] + MOV r11, #0 + UMLAL r4, r11, r12, lr + STR r4, [sp, #4] + ADDS r5, r5, r11 + /* A[1] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[1] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[1] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[1] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[1] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[1] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[1] * B[7] */ + LDR lr, [r2, #28] + ADC r4, r0, #0 + UMLAL r3, r4, r12, lr + /* A[2] * B[0] */ + LDR r12, [r1, #8] + LDR lr, [r2] + MOV r11, #0 + UMLAL r5, r11, r12, lr + STR r5, [sp, #8] + ADDS r6, r6, r11 + /* A[2] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[2] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[2] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[2] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[2] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[2] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[2] * B[7] */ + LDR lr, [r2, #28] + ADC r5, r0, #0 + UMLAL r4, r5, r12, lr + /* A[3] * B[0] */ + LDR r12, [r1, #12] + LDR lr, [r2] + MOV r11, #0 + UMLAL r6, r11, r12, lr + STR r6, [sp, #12] + ADDS r7, r7, r11 + /* A[3] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[3] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[3] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[3] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[3] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[3] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[3] * B[7] */ + LDR lr, [r2, #28] + ADC r6, r0, #0 + UMLAL r5, r6, r12, lr + /* A[4] * B[0] */ + LDR r12, [r1, #16] + LDR lr, [r2] + MOV r11, #0 + UMLAL r7, r11, r12, lr + STR r7, [sp, #16] + ADDS r8, r8, r11 + /* A[4] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[4] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[4] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[4] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[4] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[4] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[4] * B[7] */ + LDR lr, [r2, #28] + ADC r7, r0, #0 + UMLAL r6, r7, r12, lr + /* A[5] * B[0] */ + LDR r12, [r1, #20] + LDR lr, [r2] + MOV r11, #0 + UMLAL r8, r11, r12, lr + STR r8, [sp, #20] + ADDS r9, r9, r11 + /* A[5] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[5] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[5] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[5] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[5] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[5] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[5] * B[7] */ + LDR lr, [r2, #28] + ADC r8, r0, #0 + UMLAL r7, r8, r12, lr + /* A[6] * B[0] */ + LDR r12, [r1, #24] + LDR lr, [r2] + MOV r11, #0 + UMLAL r9, r11, r12, lr + STR r9, [sp, #24] + ADDS r10, r10, r11 + /* A[6] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[6] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[6] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[6] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[6] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[6] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[6] * B[7] */ + LDR lr, [r2, #28] + ADC r9, r0, #0 + UMLAL r8, r9, r12, lr + /* A[7] * B[0] */ + LDR r12, [r1, #28] + LDR lr, [r2] + MOV r11, #0 + UMLAL r10, r11, r12, lr + STR r10, [sp, #28] + ADDS r3, r3, r11 + /* A[7] * B[1] */ + LDR lr, [r2, #4] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[7] * B[2] */ + LDR lr, [r2, #8] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[7] * B[3] */ + LDR lr, [r2, #12] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[7] * B[4] */ + LDR lr, [r2, #16] + ADC r11, r0, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[7] * B[5] */ + LDR lr, [r2, #20] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[7] * B[6] */ + LDR lr, [r2, #24] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[7] * B[7] */ + LDR lr, [r2, #28] + ADC r10, r0, #0 + UMLAL r9, r10, r12, lr + /* Reduce */ + LDR r2, [sp, #28] + MOV lr, sp + MOV r12, #38 + UMULL r10, r11, r10, r12 + ADDS r10, r10, r2 + ADC r11, r11, #0 + MOV r12, #19 + LSL r11, r11, #1 + ORR r11, r11, r10, LSR #31 + MUL r11, r11, r12 + LDM lr!, {r1, r2} + MOV r12, #38 + ADDS r1, r1, r11 + ADC r11, r0, #0 + UMLAL r1, r11, r3, r12 + ADDS r2, r2, r11 + ADC r11, r0, #0 + UMLAL r2, r11, r4, r12 + LDM lr!, {r3, r4} + ADDS r3, r3, r11 + ADC r11, r0, #0 + UMLAL r3, r11, r5, r12 + ADDS r4, r4, r11 + ADC r11, r0, #0 + UMLAL r4, r11, r6, r12 + LDM lr!, {r5, r6} + ADDS r5, r5, r11 + ADC r11, r0, #0 + UMLAL r5, r11, r7, r12 + ADDS r6, r6, r11 + ADC r11, r0, #0 + UMLAL r6, r11, r8, r12 + LDM lr!, {r7, r8} + ADDS r7, r7, r11 + ADC r11, r0, #0 + UMLAL r7, r11, r9, r12 + BFC r10, #31, #1 + ADDS r8, r10, r11 + ADDS r11, r1, #19 + ADCS r11, r2, #0 + ADCS r11, r3, #0 + ADCS r11, r4, #0 + ADCS r11, r5, #0 + ADCS r11, r6, #0 + ADCS r11, r7, #0 + ADC r11, r8, #0 + ASR r11, r11, #31 + AND r11, r11, #19 + ADDS r1, r1, r11 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADCS r7, r7, #0 + ADC r8, r8, #0 + BFC r8, #31, #1 + /* Store */ + LDR r0, [sp, #36] + STM r0, {r1, r2, r3, r4, r5, r6, r7, r8} + ADD sp, sp, #40 + POP {pc} + /* Cycle Count = 425 */ + .size fe_mul_op_full_red,.-fe_mul_op_full_red #else .text .align 4 @@ -2020,6 +2417,157 @@ fe_mul_op: POP {pc} /* Cycle Count = 239 */ .size fe_mul_op,.-fe_mul_op + .text + .align 4 + .globl fe_mul_op_full_red + .type fe_mul_op_full_red, %function +fe_mul_op_full_red: + PUSH {lr} + SUB sp, sp, #44 + STRD r0, r1, [sp, #36] + MOV lr, r2 + LDM r1, {r0, r1, r2, r3} + LDM lr!, {r4, r5, r6} + UMULL r10, r11, r0, r4 + UMULL r12, r7, r1, r4 + UMAAL r11, r12, r0, r5 + UMULL r8, r9, r2, r4 + UMAAL r12, r8, r1, r5 + UMAAL r12, r7, r0, r6 + UMAAL r8, r9, r3, r4 + STM sp, {r10, r11, r12} + UMAAL r7, r8, r2, r5 + LDM lr!, {r4} + UMULL r10, r11, r1, r6 + UMAAL r8, r9, r2, r6 + UMAAL r7, r10, r0, r4 + UMAAL r8, r11, r3, r5 + STR r7, [sp, #12] + UMAAL r8, r10, r1, r4 + UMAAL r9, r11, r3, r6 + UMAAL r9, r10, r2, r4 + UMAAL r10, r11, r3, r4 + LDM lr, {r4, r5, r6, r7} + MOV r12, #0 + UMLAL r8, r12, r0, r4 + UMAAL r9, r12, r1, r4 + UMAAL r10, r12, r2, r4 + UMAAL r11, r12, r3, r4 + MOV r4, #0 + UMLAL r9, r4, r0, r5 + UMAAL r10, r4, r1, r5 + UMAAL r11, r4, r2, r5 + UMAAL r12, r4, r3, r5 + MOV r5, #0 + UMLAL r10, r5, r0, r6 + UMAAL r11, r5, r1, r6 + UMAAL r12, r5, r2, r6 + UMAAL r4, r5, r3, r6 + MOV r6, #0 + UMLAL r11, r6, r0, r7 + LDR r0, [sp, #40] + UMAAL r12, r6, r1, r7 + ADD r0, r0, #16 + UMAAL r4, r6, r2, r7 + SUB lr, lr, #16 + UMAAL r5, r6, r3, r7 + LDM r0, {r0, r1, r2, r3} + STR r6, [sp, #32] + LDM lr!, {r6} + MOV r7, #0 + UMLAL r8, r7, r0, r6 + UMAAL r9, r7, r1, r6 + STR r8, [sp, #16] + UMAAL r10, r7, r2, r6 + UMAAL r11, r7, r3, r6 + LDM lr!, {r6} + MOV r8, #0 + UMLAL r9, r8, r0, r6 + UMAAL r10, r8, r1, r6 + STR r9, [sp, #20] + UMAAL r11, r8, r2, r6 + UMAAL r12, r8, r3, r6 + LDM lr!, {r6} + MOV r9, #0 + UMLAL r10, r9, r0, r6 + UMAAL r11, r9, r1, r6 + STR r10, [sp, #24] + UMAAL r12, r9, r2, r6 + UMAAL r4, r9, r3, r6 + LDM lr!, {r6} + MOV r10, #0 + UMLAL r11, r10, r0, r6 + UMAAL r12, r10, r1, r6 + STR r11, [sp, #28] + UMAAL r4, r10, r2, r6 + UMAAL r5, r10, r3, r6 + LDM lr!, {r11} + UMAAL r12, r7, r0, r11 + UMAAL r4, r7, r1, r11 + LDR r6, [sp, #32] + UMAAL r5, r7, r2, r11 + UMAAL r6, r7, r3, r11 + LDM lr!, {r11} + UMAAL r4, r8, r0, r11 + UMAAL r5, r8, r1, r11 + UMAAL r6, r8, r2, r11 + UMAAL r7, r8, r3, r11 + LDM lr, {r11, lr} + UMAAL r5, r9, r0, r11 + UMAAL r6, r10, r0, lr + UMAAL r6, r9, r1, r11 + UMAAL r7, r10, r1, lr + UMAAL r7, r9, r2, r11 + UMAAL r8, r10, r2, lr + UMAAL r8, r9, r3, r11 + UMAAL r9, r10, r3, lr + /* Reduce */ + LDR r0, [sp, #28] + MOV lr, #37 + UMAAL r10, r0, r10, lr + MOV lr, #19 + LSL r0, r0, #1 + ORR r0, r0, r10, LSR #31 + MUL r11, r0, lr + POP {r0, r1, r2} + MOV lr, #38 + UMAAL r0, r11, r12, lr + UMAAL r1, r11, r4, lr + UMAAL r2, r11, r5, lr + POP {r3, r4, r5} + UMAAL r3, r11, r6, lr + UMAAL r4, r11, r7, lr + UMAAL r5, r11, r8, lr + POP {r6} + BFC r10, #31, #1 + UMAAL r6, r11, r9, lr + ADD r7, r10, r11 + ADDS r11, r0, #19 + ADCS r11, r1, #0 + ADCS r11, r2, #0 + ADCS r11, r3, #0 + ADCS r11, r4, #0 + ADCS r11, r5, #0 + ADCS r11, r6, #0 + ADC r11, r7, #0 + ASR r11, r11, #31 + AND r11, r11, #19 + ADDS r0, r0, r11 + ADCS r1, r1, #0 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADC r7, r7, #0 + BFC r7, #31, #1 + LDR lr, [sp, #8] + /* Store */ + STM lr, {r0, r1, r2, r3, r4, r5, r6, r7} + ADD sp, sp, #16 + POP {pc} + /* Cycle Count = 258 */ + .size fe_mul_op_full_red,.-fe_mul_op_full_red #endif /* WOLFSSL_ARM_ARCH_7M */ .text .align 4 @@ -2303,6 +2851,296 @@ fe_sq_op: POP {pc} /* Cycle Count = 355 */ .size fe_sq_op,.-fe_sq_op + .text + .align 4 + .globl fe_sq_op_full_red + .type fe_sq_op_full_red, %function +fe_sq_op_full_red: + PUSH {lr} + SUB sp, sp, #0x44 + STR r0, [sp, #64] + /* Square */ + MOV r0, #0 + LDR r12, [r1] + /* A[0] * A[1] */ + LDR lr, [r1, #4] + UMULL r4, r5, r12, lr + /* A[0] * A[3] */ + LDR lr, [r1, #12] + UMULL r6, r7, r12, lr + /* A[0] * A[5] */ + LDR lr, [r1, #20] + UMULL r8, r9, r12, lr + /* A[0] * A[7] */ + LDR lr, [r1, #28] + UMULL r10, r3, r12, lr + /* A[0] * A[2] */ + LDR lr, [r1, #8] + MOV r11, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[0] * A[4] */ + LDR lr, [r1, #16] + ADCS r7, r7, #0 + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + ADDS r8, r8, r11 + /* A[0] * A[6] */ + LDR lr, [r1, #24] + ADCS r9, r9, #0 + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + ADCS r3, r3, #0 + STR r4, [sp, #4] + STR r5, [sp, #8] + /* A[1] * A[2] */ + LDR r12, [r1, #4] + LDR lr, [r1, #8] + MOV r11, #0 + UMLAL r6, r11, r12, lr + STR r6, [sp, #12] + ADDS r7, r7, r11 + /* A[1] * A[3] */ + LDR lr, [r1, #12] + ADC r11, r0, #0 + UMLAL r7, r11, r12, lr + STR r7, [sp, #16] + ADDS r8, r8, r11 + /* A[1] * A[4] */ + LDR lr, [r1, #16] + ADC r11, r0, #0 + UMLAL r8, r11, r12, lr + ADDS r9, r9, r11 + /* A[1] * A[5] */ + LDR lr, [r1, #20] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + ADDS r10, r10, r11 + /* A[1] * A[6] */ + LDR lr, [r1, #24] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[1] * A[7] */ + LDR lr, [r1, #28] + ADC r4, r0, #0 + UMLAL r3, r4, r12, lr + /* A[2] * A[3] */ + LDR r12, [r1, #8] + LDR lr, [r1, #12] + MOV r11, #0 + UMLAL r8, r11, r12, lr + STR r8, [sp, #20] + ADDS r9, r9, r11 + /* A[2] * A[4] */ + LDR lr, [r1, #16] + ADC r11, r0, #0 + UMLAL r9, r11, r12, lr + STR r9, [sp, #24] + ADDS r10, r10, r11 + /* A[2] * A[5] */ + LDR lr, [r1, #20] + ADC r11, r0, #0 + UMLAL r10, r11, r12, lr + ADDS r3, r3, r11 + /* A[2] * A[6] */ + LDR lr, [r1, #24] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[2] * A[7] */ + LDR lr, [r1, #28] + ADC r5, r0, #0 + UMLAL r4, r5, r12, lr + /* A[3] * A[4] */ + LDR r12, [r1, #12] + LDR lr, [r1, #16] + MOV r11, #0 + UMLAL r10, r11, r12, lr + STR r10, [sp, #28] + ADDS r3, r3, r11 + /* A[3] * A[5] */ + LDR lr, [r1, #20] + ADC r11, r0, #0 + UMLAL r3, r11, r12, lr + ADDS r4, r4, r11 + /* A[3] * A[6] */ + LDR lr, [r1, #24] + ADC r11, r0, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[3] * A[7] */ + LDR lr, [r1, #28] + ADC r6, r0, #0 + UMLAL r5, r6, r12, lr + /* A[4] * A[5] */ + LDR r12, [r1, #16] + LDR lr, [r1, #20] + MOV r11, #0 + UMLAL r4, r11, r12, lr + ADDS r5, r5, r11 + /* A[4] * A[6] */ + LDR lr, [r1, #24] + ADC r11, r0, #0 + UMLAL r5, r11, r12, lr + ADDS r6, r6, r11 + /* A[4] * A[7] */ + LDR lr, [r1, #28] + ADC r7, r0, #0 + UMLAL r6, r7, r12, lr + /* A[5] * A[6] */ + LDR r12, [r1, #20] + LDR lr, [r1, #24] + MOV r11, #0 + UMLAL r6, r11, r12, lr + ADDS r7, r7, r11 + /* A[5] * A[7] */ + LDR lr, [r1, #28] + ADC r8, r0, #0 + UMLAL r7, r8, r12, lr + /* A[6] * A[7] */ + LDR r12, [r1, #24] + LDR lr, [r1, #28] + MOV r9, #0 + UMLAL r8, r9, r12, lr + ADD lr, sp, #32 + STM lr, {r3, r4, r5, r6, r7, r8, r9} + ADD lr, sp, #4 + LDM lr, {r4, r5, r6, r7, r8, r9, r10} + ADDS r4, r4, r4 + ADCS r5, r5, r5 + ADCS r6, r6, r6 + ADCS r7, r7, r7 + ADCS r8, r8, r8 + ADCS r9, r9, r9 + ADCS r10, r10, r10 + STM lr!, {r4, r5, r6, r7, r8, r9, r10} + LDM lr, {r3, r4, r5, r6, r7, r8, r9} + ADCS r3, r3, r3 + ADCS r4, r4, r4 + ADCS r5, r5, r5 + ADCS r6, r6, r6 + ADCS r7, r7, r7 + ADCS r8, r8, r8 + ADCS r9, r9, r9 + ADC r10, r0, #0 + STM lr, {r3, r4, r5, r6, r7, r8, r9, r10} + ADD lr, sp, #4 + LDM lr, {r4, r5, r6, r7, r8, r9, r10} + MOV lr, sp + /* A[0] * A[0] */ + LDR r12, [r1] + UMULL r3, r11, r12, r12 + ADDS r4, r4, r11 + /* A[1] * A[1] */ + LDR r12, [r1, #4] + ADCS r5, r5, #0 + ADC r11, r0, #0 + UMLAL r5, r11, r12, r12 + ADDS r6, r6, r11 + /* A[2] * A[2] */ + LDR r12, [r1, #8] + ADCS r7, r7, #0 + ADC r11, r0, #0 + UMLAL r7, r11, r12, r12 + ADDS r8, r8, r11 + /* A[3] * A[3] */ + LDR r12, [r1, #12] + ADCS r9, r9, #0 + ADC r11, r0, #0 + UMLAL r9, r11, r12, r12 + ADDS r10, r10, r11 + STM lr!, {r3, r4, r5, r6, r7, r8, r9, r10} + LDM lr, {r3, r4, r5, r6, r7, r8, r9, r10} + /* A[4] * A[4] */ + LDR r12, [r1, #16] + ADCS r3, r3, #0 + ADC r11, r0, #0 + UMLAL r3, r11, r12, r12 + ADDS r4, r4, r11 + /* A[5] * A[5] */ + LDR r12, [r1, #20] + ADCS r5, r5, #0 + ADC r11, r0, #0 + UMLAL r5, r11, r12, r12 + ADDS r6, r6, r11 + /* A[6] * A[6] */ + LDR r12, [r1, #24] + ADCS r7, r7, #0 + ADC r11, r0, #0 + UMLAL r7, r11, r12, r12 + ADDS r8, r8, r11 + /* A[7] * A[7] */ + LDR r12, [r1, #28] + ADCS r9, r9, #0 + ADC r10, r10, #0 + UMLAL r9, r10, r12, r12 + /* Reduce */ + LDR r2, [sp, #28] + MOV lr, sp + MOV r12, #38 + UMULL r10, r11, r10, r12 + ADDS r10, r10, r2 + ADC r11, r11, #0 + MOV r12, #19 + LSL r11, r11, #1 + ORR r11, r11, r10, LSR #31 + MUL r11, r11, r12 + LDM lr!, {r1, r2} + MOV r12, #38 + ADDS r1, r1, r11 + ADC r11, r0, #0 + UMLAL r1, r11, r3, r12 + ADDS r2, r2, r11 + ADC r11, r0, #0 + UMLAL r2, r11, r4, r12 + LDM lr!, {r3, r4} + ADDS r3, r3, r11 + ADC r11, r0, #0 + UMLAL r3, r11, r5, r12 + ADDS r4, r4, r11 + ADC r11, r0, #0 + UMLAL r4, r11, r6, r12 + LDM lr!, {r5, r6} + ADDS r5, r5, r11 + ADC r11, r0, #0 + UMLAL r5, r11, r7, r12 + ADDS r6, r6, r11 + ADC r11, r0, #0 + UMLAL r6, r11, r8, r12 + LDM lr!, {r7, r8} + ADDS r7, r7, r11 + ADC r11, r0, #0 + UMLAL r7, r11, r9, r12 + BFC r10, #31, #1 + ADDS r8, r10, r11 + ADDS r11, r1, #19 + ADCS r11, r2, #0 + ADCS r11, r3, #0 + ADCS r11, r4, #0 + ADCS r11, r5, #0 + ADCS r11, r6, #0 + ADCS r11, r7, #0 + ADC r11, r8, #0 + ASR r11, r11, #31 + AND r11, r11, #19 + ADDS r1, r1, r11 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADCS r7, r7, #0 + ADC r8, r8, #0 + BFC r8, #31, #1 + /* Store */ + LDR r0, [sp, #64] + STM r0, {r1, r2, r3, r4, r5, r6, r7, r8} + ADD sp, sp, #0x44 + POP {pc} + /* Cycle Count = 374 */ + .size fe_sq_op_full_red,.-fe_sq_op_full_red #else .text .align 4 @@ -2422,6 +3260,143 @@ fe_sq_op: POP {pc} /* Cycle Count = 179 */ .size fe_sq_op,.-fe_sq_op + .text + .align 4 + .globl fe_sq_op_full_red + .type fe_sq_op_full_red, %function +fe_sq_op_full_red: + PUSH {lr} + SUB sp, sp, #32 + STR r0, [sp, #28] + LDM r1, {r0, r1, r2, r3, r4, r5, r6, r7} + /* Square */ + UMULL r9, r10, r0, r0 + UMULL r11, r12, r0, r1 + ADDS r11, r11, r11 + MOV lr, #0 + UMAAL r10, r11, lr, lr + STM sp, {r9, r10} + MOV r8, lr + UMAAL r8, r12, r0, r2 + ADCS r8, r8, r8 + UMAAL r8, r11, r1, r1 + UMULL r9, r10, r0, r3 + UMAAL r9, r12, r1, r2 + ADCS r9, r9, r9 + UMAAL r9, r11, lr, lr + STRD r8, r9, [sp, #8] + MOV r9, lr + UMAAL r9, r10, r0, r4 + UMAAL r9, r12, r1, r3 + ADCS r9, r9, r9 + UMAAL r9, r11, r2, r2 + STR r9, [sp, #16] + UMULL r9, r8, r0, r5 + UMAAL r9, r12, r1, r4 + UMAAL r9, r10, r2, r3 + ADCS r9, r9, r9 + UMAAL r9, r11, lr, lr + STR r9, [sp, #20] + MOV r9, lr + UMAAL r9, r8, r0, r6 + UMAAL r9, r12, r1, r5 + UMAAL r9, r10, r2, r4 + ADCS r9, r9, r9 + UMAAL r9, r11, r3, r3 + STR r9, [sp, #24] + UMULL r0, r9, r0, r7 + UMAAL r0, r8, r1, r6 + UMAAL r0, r12, r2, r5 + UMAAL r0, r10, r3, r4 + ADCS r0, r0, r0 + UMAAL r0, r11, lr, lr + /* R[7] = r0 */ + UMAAL r9, r8, r1, r7 + UMAAL r9, r10, r2, r6 + UMAAL r12, r9, r3, r5 + ADCS r12, r12, r12 + UMAAL r12, r11, r4, r4 + /* R[8] = r12 */ + UMAAL r9, r8, r2, r7 + UMAAL r10, r9, r3, r6 + MOV r2, lr + UMAAL r10, r2, r4, r5 + ADCS r10, r10, r10 + UMAAL r11, r10, lr, lr + /* R[9] = r11 */ + UMAAL r2, r8, r3, r7 + UMAAL r2, r9, r4, r6 + ADCS r3, r2, r2 + UMAAL r10, r3, r5, r5 + /* R[10] = r10 */ + MOV r1, lr + UMAAL r1, r8, r4, r7 + UMAAL r1, r9, r5, r6 + ADCS r4, r1, r1 + UMAAL r3, r4, lr, lr + /* R[11] = r3 */ + UMAAL r8, r9, r5, r7 + ADCS r8, r8, r8 + UMAAL r4, r8, r6, r6 + /* R[12] = r4 */ + MOV r5, lr + UMAAL r5, r9, r6, r7 + ADCS r5, r5, r5 + UMAAL r8, r5, lr, lr + /* R[13] = r8 */ + ADCS r9, r9, r9 + UMAAL r9, r5, r7, r7 + ADCS r7, r5, lr + /* R[14] = r9 */ + /* R[15] = r7 */ + /* Reduce */ + MOV r6, #37 + UMAAL r7, r0, r7, r6 + MOV r6, #19 + LSL r0, r0, #1 + ORR r0, r0, r7, LSR #31 + MUL lr, r0, r6 + POP {r0, r1} + MOV r6, #38 + UMAAL r0, lr, r12, r6 + UMAAL r1, lr, r11, r6 + MOV r12, r3 + MOV r11, r4 + POP {r2, r3, r4} + UMAAL r2, lr, r10, r6 + UMAAL r3, lr, r12, r6 + UMAAL r4, lr, r11, r6 + MOV r12, r6 + POP {r5, r6} + UMAAL r5, lr, r8, r12 + BFC r7, #31, #1 + UMAAL r6, lr, r9, r12 + ADD r7, r7, lr + ADDS lr, r0, #19 + ADCS lr, r1, #0 + ADCS lr, r2, #0 + ADCS lr, r3, #0 + ADCS lr, r4, #0 + ADCS lr, r5, #0 + ADCS lr, r6, #0 + ADC lr, r7, #0 + ASR lr, lr, #31 + AND lr, lr, #19 + ADDS r0, r0, lr + ADCS r1, r1, #0 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADC r7, r7, #0 + BFC r7, #31, #1 + POP {lr} + /* Store */ + STM lr, {r0, r1, r2, r3, r4, r5, r6, r7} + POP {pc} + /* Cycle Count = 198 */ + .size fe_sq_op_full_red,.-fe_sq_op_full_red #endif /* WOLFSSL_ARM_ARCH_7M */ .text .align 4 @@ -2689,17 +3664,17 @@ L_curve25519_bits: LDR r2, [sp, #160] ADD r1, sp, #0x60 ADD r0, sp, #32 - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x80 MOV r1, sp MOV r0, sp - BL fe_mul_op + BL fe_mul_op_full_red ADD r1, sp, #0x80 ADD r0, sp, #0x80 - BL fe_sq_op + BL fe_sq_op_full_red LDR r1, [sp, #160] ADD r0, sp, #0x60 - BL fe_sq_op + BL fe_sq_op_full_red MOV r3, sp ADD r2, sp, #32 MOV r1, sp @@ -2708,7 +3683,7 @@ L_curve25519_bits: ADD r2, sp, #0x80 ADD r1, sp, #0x60 LDR r0, [sp, #160] - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x80 ADD r1, sp, #0x60 ADD r0, sp, #0x60 @@ -2721,7 +3696,7 @@ L_curve25519_bits: BL fe_mul121666 ADD r1, sp, #0x40 ADD r0, sp, #0x40 - BL fe_sq_op + BL fe_sq_op_full_red ADD r2, sp, #32 ADD r1, sp, #0x80 ADD r0, sp, #0x80 @@ -2729,11 +3704,11 @@ L_curve25519_bits: MOV r2, sp LDR r1, [sp, #168] ADD r0, sp, #32 - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x80 ADD r1, sp, #0x60 MOV r0, sp - BL fe_mul_op + BL fe_mul_op_full_red LDR r2, [sp, #176] LDR r1, [sp, #180] SUBS r1, r1, #1 @@ -3035,17 +4010,17 @@ L_curve25519_bits: LDR r2, [sp, #176] ADD r1, sp, #0x60 LDR r0, [sp, #188] - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x80 LDR r1, [sp, #184] LDR r0, [sp, #184] - BL fe_mul_op + BL fe_mul_op_full_red ADD r1, sp, #0x80 ADD r0, sp, #0x60 - BL fe_sq_op + BL fe_sq_op_full_red LDR r1, [sp, #176] ADD r0, sp, #0x80 - BL fe_sq_op + BL fe_sq_op_full_red LDR r3, [sp, #184] LDR r2, [sp, #188] LDR r1, [sp, #184] @@ -3054,7 +4029,7 @@ L_curve25519_bits: ADD r2, sp, #0x60 ADD r1, sp, #0x80 LDR r0, [sp, #176] - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x60 ADD r1, sp, #0x80 ADD r0, sp, #0x80 @@ -3067,7 +4042,7 @@ L_curve25519_bits: BL fe_mul121666 LDR r1, [sp, #180] LDR r0, [sp, #180] - BL fe_sq_op + BL fe_sq_op_full_red LDR r2, [sp, #188] ADD r1, sp, #0x60 ADD r0, sp, #0x60 @@ -3075,11 +4050,11 @@ L_curve25519_bits: LDR r2, [sp, #184] LDR r1, [sp, #172] LDR r0, [sp, #188] - BL fe_mul_op + BL fe_mul_op_full_red ADD r2, sp, #0x60 ADD r1, sp, #0x80 LDR r0, [sp, #184] - BL fe_mul_op + BL fe_mul_op_full_red LDR r2, [sp, #168] SUBS r2, r2, #1 #if defined(__GNUC__) || defined(__ICCARM__) || defined(__IAR_SYSTEMS_ICC__) @@ -3792,12 +4767,31 @@ fe_sq2: BFC r8, #31, #1 ADCS r7, r7, #0 ADC r8, r8, #0 + ADDS r11, r1, #19 + ADCS r11, r2, #0 + ADCS r11, r3, #0 + ADCS r11, r4, #0 + ADCS r11, r5, #0 + ADCS r11, r6, #0 + ADCS r11, r7, #0 + ADC r11, r8, #0 + ASR r11, r11, #31 + AND r11, r11, #19 + ADDS r1, r1, r11 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADCS r7, r7, #0 + ADC r8, r8, #0 + BFC r8, #31, #1 /* Store */ LDR r0, [sp, #64] STM r0, {r1, r2, r3, r4, r5, r6, r7, r8} ADD sp, sp, #0x44 POP {pc} - /* Cycle Count = 385 */ + /* Cycle Count = 404 */ .size fe_sq2,.-fe_sq2 #else .text @@ -3945,13 +4939,32 @@ fe_sq2: BFC r7, #31, #1 ADCS r6, r6, #0 ADC r7, r7, #0 + ADDS r12, r0, #19 + ADCS r12, r1, #0 + ADCS r12, r2, #0 + ADCS r12, r3, #0 + ADCS r12, r4, #0 + ADCS r12, r5, #0 + ADCS r12, r6, #0 + ADC r12, r7, #0 + ASR r12, r12, #31 + AND r12, r12, #19 + ADDS r0, r0, r12 + ADCS r1, r1, #0 + ADCS r2, r2, #0 + ADCS r3, r3, #0 + ADCS r4, r4, #0 + ADCS r5, r5, #0 + ADCS r6, r6, #0 + ADC r7, r7, #0 + BFC r7, #31, #1 POP {r12, lr} /* Store */ STM r12, {r0, r1, r2, r3, r4, r5, r6, r7} MOV r0, r12 MOV r1, lr POP {pc} - /* Cycle Count = 213 */ + /* Cycle Count = 232 */ .size fe_sq2,.-fe_sq2 #endif /* WOLFSSL_ARM_ARCH_7M */ .text diff --git a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c index e023636c44..aff1997f0c 100644 --- a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c +++ b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c @@ -59,9 +59,9 @@ #if !defined(CURVE25519_SMALL) || !defined(ED25519_SMALL) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_init() +WC_OMIT_FRAME_POINTER void fe_init(void) #else -WC_OMIT_FRAME_POINTER void fe_init() +WC_OMIT_FRAME_POINTER void fe_init(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -81,9 +81,9 @@ WC_OMIT_FRAME_POINTER void fe_init() void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op() +WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op() +WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -189,9 +189,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op() void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op() +WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op() +WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -261,9 +261,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op() +WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op() +WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -1763,9 +1763,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2151,12 +2151,421 @@ WC_OMIT_FRAME_POINTER void fe_mul_op() ); } +void fe_mul_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "SUB sp, sp, #40\n\t" + "STR r0, [sp, #36]\n\t" + "MOV r0, #0\n\t" + "LDR r12, [r1]\n\t" + /* A[0] * B[0] */ + "LDR lr, [r2]\n\t" + "UMULL r3, r4, r12, lr\n\t" + /* A[0] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "UMULL r5, r6, r12, lr\n\t" + /* A[0] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "UMULL r7, r8, r12, lr\n\t" + /* A[0] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "UMULL r9, r10, r12, lr\n\t" + "STR r3, [sp]\n\t" + /* A[0] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "MOV r11, r0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[0] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADCS r6, r6, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[0] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADCS r8, r8, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[0] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADCS r10, r10, #0\n\t" + "ADC r3, r0, #0\n\t" + "UMLAL r10, r3, r12, lr\n\t" + /* A[1] * B[0] */ + "LDR r12, [r1, #4]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "STR r4, [sp, #4]\n\t" + "ADDS r5, r5, r11\n\t" + /* A[1] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[1] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[1] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[1] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[1] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[1] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[1] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r4, r0, #0\n\t" + "UMLAL r3, r4, r12, lr\n\t" + /* A[2] * B[0] */ + "LDR r12, [r1, #8]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "STR r5, [sp, #8]\n\t" + "ADDS r6, r6, r11\n\t" + /* A[2] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[2] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[2] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[2] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[2] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[2] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[2] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r5, r0, #0\n\t" + "UMLAL r4, r5, r12, lr\n\t" + /* A[3] * B[0] */ + "LDR r12, [r1, #12]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "STR r6, [sp, #12]\n\t" + "ADDS r7, r7, r11\n\t" + /* A[3] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[3] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[3] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[3] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[3] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[3] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[3] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r6, r0, #0\n\t" + "UMLAL r5, r6, r12, lr\n\t" + /* A[4] * B[0] */ + "LDR r12, [r1, #16]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "STR r7, [sp, #16]\n\t" + "ADDS r8, r8, r11\n\t" + /* A[4] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[4] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[4] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[4] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[4] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[4] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[4] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r7, r0, #0\n\t" + "UMLAL r6, r7, r12, lr\n\t" + /* A[5] * B[0] */ + "LDR r12, [r1, #20]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "STR r8, [sp, #20]\n\t" + "ADDS r9, r9, r11\n\t" + /* A[5] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[5] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[5] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[5] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[5] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[5] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[5] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r8, r0, #0\n\t" + "UMLAL r7, r8, r12, lr\n\t" + /* A[6] * B[0] */ + "LDR r12, [r1, #24]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "STR r9, [sp, #24]\n\t" + "ADDS r10, r10, r11\n\t" + /* A[6] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[6] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[6] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[6] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[6] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[6] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[6] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r9, r0, #0\n\t" + "UMLAL r8, r9, r12, lr\n\t" + /* A[7] * B[0] */ + "LDR r12, [r1, #28]\n\t" + "LDR lr, [r2]\n\t" + "MOV r11, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "STR r10, [sp, #28]\n\t" + "ADDS r3, r3, r11\n\t" + /* A[7] * B[1] */ + "LDR lr, [r2, #4]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[7] * B[2] */ + "LDR lr, [r2, #8]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[7] * B[3] */ + "LDR lr, [r2, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[7] * B[4] */ + "LDR lr, [r2, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[7] * B[5] */ + "LDR lr, [r2, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[7] * B[6] */ + "LDR lr, [r2, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[7] * B[7] */ + "LDR lr, [r2, #28]\n\t" + "ADC r10, r0, #0\n\t" + "UMLAL r9, r10, r12, lr\n\t" + /* Reduce */ + "LDR r2, [sp, #28]\n\t" + "MOV lr, sp\n\t" + "MOV r12, #38\n\t" + "UMULL r10, r11, r10, r12\n\t" + "ADDS r10, r10, r2\n\t" + "ADC r11, r11, #0\n\t" + "MOV r12, #19\n\t" + "LSL r11, r11, #1\n\t" + "ORR r11, r11, r10, LSR #31\n\t" + "MUL r11, r11, r12\n\t" + "LDM lr!, {r1, r2}\n\t" + "MOV r12, #38\n\t" + "ADDS r1, r1, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r1, r11, r3, r12\n\t" + "ADDS r2, r2, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r2, r11, r4, r12\n\t" + "LDM lr!, {r3, r4}\n\t" + "ADDS r3, r3, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r5, r12\n\t" + "ADDS r4, r4, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r6, r12\n\t" + "LDM lr!, {r5, r6}\n\t" + "ADDS r5, r5, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r7, r12\n\t" + "ADDS r6, r6, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r8, r12\n\t" + "LDM lr!, {r7, r8}\n\t" + "ADDS r7, r7, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r9, r12\n\t" + "BFC r10, #31, #1\n\t" + "ADDS r8, r10, r11\n\t" + "ADDS r11, r1, #19\n\t" + "ADCS r11, r2, #0\n\t" + "ADCS r11, r3, #0\n\t" + "ADCS r11, r4, #0\n\t" + "ADCS r11, r5, #0\n\t" + "ADCS r11, r6, #0\n\t" + "ADCS r11, r7, #0\n\t" + "ADC r11, r8, #0\n\t" + "ASR r11, r11, #31\n\t" + "AND r11, r11, #19\n\t" + "ADDS r1, r1, r11\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r8, r8, #0\n\t" + "BFC r8, #31, #1\n\t" + /* Store */ + "LDR r0, [sp, #36]\n\t" + "STM r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" + "ADD sp, sp, #40\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op() +WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2296,6 +2705,169 @@ WC_OMIT_FRAME_POINTER void fe_mul_op() ); } +void fe_mul_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "SUB sp, sp, #44\n\t" + "STRD r0, r1, [sp, #36]\n\t" + "MOV lr, r2\n\t" + "LDM r1, {r0, r1, r2, r3}\n\t" + "LDM lr!, {r4, r5, r6}\n\t" + "UMULL r10, r11, r0, r4\n\t" + "UMULL r12, r7, r1, r4\n\t" + "UMAAL r11, r12, r0, r5\n\t" + "UMULL r8, r9, r2, r4\n\t" + "UMAAL r12, r8, r1, r5\n\t" + "UMAAL r12, r7, r0, r6\n\t" + "UMAAL r8, r9, r3, r4\n\t" + "STM sp, {r10, r11, r12}\n\t" + "UMAAL r7, r8, r2, r5\n\t" + "LDM lr!, {r4}\n\t" + "UMULL r10, r11, r1, r6\n\t" + "UMAAL r8, r9, r2, r6\n\t" + "UMAAL r7, r10, r0, r4\n\t" + "UMAAL r8, r11, r3, r5\n\t" + "STR r7, [sp, #12]\n\t" + "UMAAL r8, r10, r1, r4\n\t" + "UMAAL r9, r11, r3, r6\n\t" + "UMAAL r9, r10, r2, r4\n\t" + "UMAAL r10, r11, r3, r4\n\t" + "LDM lr, {r4, r5, r6, r7}\n\t" + "MOV r12, #0\n\t" + "UMLAL r8, r12, r0, r4\n\t" + "UMAAL r9, r12, r1, r4\n\t" + "UMAAL r10, r12, r2, r4\n\t" + "UMAAL r11, r12, r3, r4\n\t" + "MOV r4, #0\n\t" + "UMLAL r9, r4, r0, r5\n\t" + "UMAAL r10, r4, r1, r5\n\t" + "UMAAL r11, r4, r2, r5\n\t" + "UMAAL r12, r4, r3, r5\n\t" + "MOV r5, #0\n\t" + "UMLAL r10, r5, r0, r6\n\t" + "UMAAL r11, r5, r1, r6\n\t" + "UMAAL r12, r5, r2, r6\n\t" + "UMAAL r4, r5, r3, r6\n\t" + "MOV r6, #0\n\t" + "UMLAL r11, r6, r0, r7\n\t" + "LDR r0, [sp, #40]\n\t" + "UMAAL r12, r6, r1, r7\n\t" + "ADD r0, r0, #16\n\t" + "UMAAL r4, r6, r2, r7\n\t" + "SUB lr, lr, #16\n\t" + "UMAAL r5, r6, r3, r7\n\t" + "LDM r0, {r0, r1, r2, r3}\n\t" + "STR r6, [sp, #32]\n\t" + "LDM lr!, {r6}\n\t" + "MOV r7, #0\n\t" + "UMLAL r8, r7, r0, r6\n\t" + "UMAAL r9, r7, r1, r6\n\t" + "STR r8, [sp, #16]\n\t" + "UMAAL r10, r7, r2, r6\n\t" + "UMAAL r11, r7, r3, r6\n\t" + "LDM lr!, {r6}\n\t" + "MOV r8, #0\n\t" + "UMLAL r9, r8, r0, r6\n\t" + "UMAAL r10, r8, r1, r6\n\t" + "STR r9, [sp, #20]\n\t" + "UMAAL r11, r8, r2, r6\n\t" + "UMAAL r12, r8, r3, r6\n\t" + "LDM lr!, {r6}\n\t" + "MOV r9, #0\n\t" + "UMLAL r10, r9, r0, r6\n\t" + "UMAAL r11, r9, r1, r6\n\t" + "STR r10, [sp, #24]\n\t" + "UMAAL r12, r9, r2, r6\n\t" + "UMAAL r4, r9, r3, r6\n\t" + "LDM lr!, {r6}\n\t" + "MOV r10, #0\n\t" + "UMLAL r11, r10, r0, r6\n\t" + "UMAAL r12, r10, r1, r6\n\t" + "STR r11, [sp, #28]\n\t" + "UMAAL r4, r10, r2, r6\n\t" + "UMAAL r5, r10, r3, r6\n\t" + "LDM lr!, {r11}\n\t" + "UMAAL r12, r7, r0, r11\n\t" + "UMAAL r4, r7, r1, r11\n\t" + "LDR r6, [sp, #32]\n\t" + "UMAAL r5, r7, r2, r11\n\t" + "UMAAL r6, r7, r3, r11\n\t" + "LDM lr!, {r11}\n\t" + "UMAAL r4, r8, r0, r11\n\t" + "UMAAL r5, r8, r1, r11\n\t" + "UMAAL r6, r8, r2, r11\n\t" + "UMAAL r7, r8, r3, r11\n\t" + "LDM lr, {r11, lr}\n\t" + "UMAAL r5, r9, r0, r11\n\t" + "UMAAL r6, r10, r0, lr\n\t" + "UMAAL r6, r9, r1, r11\n\t" + "UMAAL r7, r10, r1, lr\n\t" + "UMAAL r7, r9, r2, r11\n\t" + "UMAAL r8, r10, r2, lr\n\t" + "UMAAL r8, r9, r3, r11\n\t" + "UMAAL r9, r10, r3, lr\n\t" + /* Reduce */ + "LDR r0, [sp, #28]\n\t" + "MOV lr, #37\n\t" + "UMAAL r10, r0, r10, lr\n\t" + "MOV lr, #19\n\t" + "LSL r0, r0, #1\n\t" + "ORR r0, r0, r10, LSR #31\n\t" + "MUL r11, r0, lr\n\t" + "POP {r0, r1, r2}\n\t" + "MOV lr, #38\n\t" + "UMAAL r0, r11, r12, lr\n\t" + "UMAAL r1, r11, r4, lr\n\t" + "UMAAL r2, r11, r5, lr\n\t" + "POP {r3, r4, r5}\n\t" + "UMAAL r3, r11, r6, lr\n\t" + "UMAAL r4, r11, r7, lr\n\t" + "UMAAL r5, r11, r8, lr\n\t" + "POP {r6}\n\t" + "BFC r10, #31, #1\n\t" + "UMAAL r6, r11, r9, lr\n\t" + "ADD r7, r10, r11\n\t" + "ADDS r11, r0, #19\n\t" + "ADCS r11, r1, #0\n\t" + "ADCS r11, r2, #0\n\t" + "ADCS r11, r3, #0\n\t" + "ADCS r11, r4, #0\n\t" + "ADCS r11, r5, #0\n\t" + "ADCS r11, r6, #0\n\t" + "ADC r11, r7, #0\n\t" + "ASR r11, r11, #31\n\t" + "AND r11, r11, #19\n\t" + "ADDS r0, r0, r11\n\t" + "ADCS r1, r1, #0\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADC r7, r7, #0\n\t" + "BFC r7, #31, #1\n\t" + "LDR lr, [sp, #8]\n\t" + /* Store */ + "STM lr, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" + "ADD sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #endif /* WOLFSSL_ARM_ARCH_7M */ #ifndef WOLFSSL_NO_VAR_ASSIGN_REG WC_OMIT_FRAME_POINTER void fe_mul(fe r_p, const fe a_p, const fe b_p) @@ -2326,9 +2898,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2607,12 +3179,314 @@ WC_OMIT_FRAME_POINTER void fe_sq_op() ); } +void fe_sq_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "SUB sp, sp, #0x44\n\t" + "STR r0, [sp, #64]\n\t" + /* Square */ + "MOV r0, #0\n\t" + "LDR r12, [r1]\n\t" + /* A[0] * A[1] */ + "LDR lr, [r1, #4]\n\t" + "UMULL r4, r5, r12, lr\n\t" + /* A[0] * A[3] */ + "LDR lr, [r1, #12]\n\t" + "UMULL r6, r7, r12, lr\n\t" + /* A[0] * A[5] */ + "LDR lr, [r1, #20]\n\t" + "UMULL r8, r9, r12, lr\n\t" + /* A[0] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "UMULL r10, r3, r12, lr\n\t" + /* A[0] * A[2] */ + "LDR lr, [r1, #8]\n\t" + "MOV r11, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[0] * A[4] */ + "LDR lr, [r1, #16]\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "ADDS r8, r8, r11\n\t" + /* A[0] * A[6] */ + "LDR lr, [r1, #24]\n\t" + "ADCS r9, r9, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + "ADCS r3, r3, #0\n\t" + "STR r4, [sp, #4]\n\t" + "STR r5, [sp, #8]\n\t" + /* A[1] * A[2] */ + "LDR r12, [r1, #4]\n\t" + "LDR lr, [r1, #8]\n\t" + "MOV r11, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "STR r6, [sp, #12]\n\t" + "ADDS r7, r7, r11\n\t" + /* A[1] * A[3] */ + "LDR lr, [r1, #12]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, lr\n\t" + "STR r7, [sp, #16]\n\t" + "ADDS r8, r8, r11\n\t" + /* A[1] * A[4] */ + "LDR lr, [r1, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "ADDS r9, r9, r11\n\t" + /* A[1] * A[5] */ + "LDR lr, [r1, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "ADDS r10, r10, r11\n\t" + /* A[1] * A[6] */ + "LDR lr, [r1, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[1] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "ADC r4, r0, #0\n\t" + "UMLAL r3, r4, r12, lr\n\t" + /* A[2] * A[3] */ + "LDR r12, [r1, #8]\n\t" + "LDR lr, [r1, #12]\n\t" + "MOV r11, #0\n\t" + "UMLAL r8, r11, r12, lr\n\t" + "STR r8, [sp, #20]\n\t" + "ADDS r9, r9, r11\n\t" + /* A[2] * A[4] */ + "LDR lr, [r1, #16]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, lr\n\t" + "STR r9, [sp, #24]\n\t" + "ADDS r10, r10, r11\n\t" + /* A[2] * A[5] */ + "LDR lr, [r1, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "ADDS r3, r3, r11\n\t" + /* A[2] * A[6] */ + "LDR lr, [r1, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[2] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "ADC r5, r0, #0\n\t" + "UMLAL r4, r5, r12, lr\n\t" + /* A[3] * A[4] */ + "LDR r12, [r1, #12]\n\t" + "LDR lr, [r1, #16]\n\t" + "MOV r11, #0\n\t" + "UMLAL r10, r11, r12, lr\n\t" + "STR r10, [sp, #28]\n\t" + "ADDS r3, r3, r11\n\t" + /* A[3] * A[5] */ + "LDR lr, [r1, #20]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, lr\n\t" + "ADDS r4, r4, r11\n\t" + /* A[3] * A[6] */ + "LDR lr, [r1, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[3] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "ADC r6, r0, #0\n\t" + "UMLAL r5, r6, r12, lr\n\t" + /* A[4] * A[5] */ + "LDR r12, [r1, #16]\n\t" + "LDR lr, [r1, #20]\n\t" + "MOV r11, #0\n\t" + "UMLAL r4, r11, r12, lr\n\t" + "ADDS r5, r5, r11\n\t" + /* A[4] * A[6] */ + "LDR lr, [r1, #24]\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, lr\n\t" + "ADDS r6, r6, r11\n\t" + /* A[4] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "ADC r7, r0, #0\n\t" + "UMLAL r6, r7, r12, lr\n\t" + /* A[5] * A[6] */ + "LDR r12, [r1, #20]\n\t" + "LDR lr, [r1, #24]\n\t" + "MOV r11, #0\n\t" + "UMLAL r6, r11, r12, lr\n\t" + "ADDS r7, r7, r11\n\t" + /* A[5] * A[7] */ + "LDR lr, [r1, #28]\n\t" + "ADC r8, r0, #0\n\t" + "UMLAL r7, r8, r12, lr\n\t" + /* A[6] * A[7] */ + "LDR r12, [r1, #24]\n\t" + "LDR lr, [r1, #28]\n\t" + "MOV r9, #0\n\t" + "UMLAL r8, r9, r12, lr\n\t" + "ADD lr, sp, #32\n\t" + "STM lr, {r3, r4, r5, r6, r7, r8, r9}\n\t" + "ADD lr, sp, #4\n\t" + "LDM lr, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "ADDS r4, r4, r4\n\t" + "ADCS r5, r5, r5\n\t" + "ADCS r6, r6, r6\n\t" + "ADCS r7, r7, r7\n\t" + "ADCS r8, r8, r8\n\t" + "ADCS r9, r9, r9\n\t" + "ADCS r10, r10, r10\n\t" + "STM lr!, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "LDM lr, {r3, r4, r5, r6, r7, r8, r9}\n\t" + "ADCS r3, r3, r3\n\t" + "ADCS r4, r4, r4\n\t" + "ADCS r5, r5, r5\n\t" + "ADCS r6, r6, r6\n\t" + "ADCS r7, r7, r7\n\t" + "ADCS r8, r8, r8\n\t" + "ADCS r9, r9, r9\n\t" + "ADC r10, r0, #0\n\t" + "STM lr, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + "ADD lr, sp, #4\n\t" + "LDM lr, {r4, r5, r6, r7, r8, r9, r10}\n\t" + "MOV lr, sp\n\t" + /* A[0] * A[0] */ + "LDR r12, [r1]\n\t" + "UMULL r3, r11, r12, r12\n\t" + "ADDS r4, r4, r11\n\t" + /* A[1] * A[1] */ + "LDR r12, [r1, #4]\n\t" + "ADCS r5, r5, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, r12\n\t" + "ADDS r6, r6, r11\n\t" + /* A[2] * A[2] */ + "LDR r12, [r1, #8]\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, r12\n\t" + "ADDS r8, r8, r11\n\t" + /* A[3] * A[3] */ + "LDR r12, [r1, #12]\n\t" + "ADCS r9, r9, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r9, r11, r12, r12\n\t" + "ADDS r10, r10, r11\n\t" + "STM lr!, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + "LDM lr, {r3, r4, r5, r6, r7, r8, r9, r10}\n\t" + /* A[4] * A[4] */ + "LDR r12, [r1, #16]\n\t" + "ADCS r3, r3, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r12, r12\n\t" + "ADDS r4, r4, r11\n\t" + /* A[5] * A[5] */ + "LDR r12, [r1, #20]\n\t" + "ADCS r5, r5, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r12, r12\n\t" + "ADDS r6, r6, r11\n\t" + /* A[6] * A[6] */ + "LDR r12, [r1, #24]\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r12, r12\n\t" + "ADDS r8, r8, r11\n\t" + /* A[7] * A[7] */ + "LDR r12, [r1, #28]\n\t" + "ADCS r9, r9, #0\n\t" + "ADC r10, r10, #0\n\t" + "UMLAL r9, r10, r12, r12\n\t" + /* Reduce */ + "LDR r2, [sp, #28]\n\t" + "MOV lr, sp\n\t" + "MOV r12, #38\n\t" + "UMULL r10, r11, r10, r12\n\t" + "ADDS r10, r10, r2\n\t" + "ADC r11, r11, #0\n\t" + "MOV r12, #19\n\t" + "LSL r11, r11, #1\n\t" + "ORR r11, r11, r10, LSR #31\n\t" + "MUL r11, r11, r12\n\t" + "LDM lr!, {r1, r2}\n\t" + "MOV r12, #38\n\t" + "ADDS r1, r1, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r1, r11, r3, r12\n\t" + "ADDS r2, r2, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r2, r11, r4, r12\n\t" + "LDM lr!, {r3, r4}\n\t" + "ADDS r3, r3, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r3, r11, r5, r12\n\t" + "ADDS r4, r4, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r4, r11, r6, r12\n\t" + "LDM lr!, {r5, r6}\n\t" + "ADDS r5, r5, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r5, r11, r7, r12\n\t" + "ADDS r6, r6, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r6, r11, r8, r12\n\t" + "LDM lr!, {r7, r8}\n\t" + "ADDS r7, r7, r11\n\t" + "ADC r11, r0, #0\n\t" + "UMLAL r7, r11, r9, r12\n\t" + "BFC r10, #31, #1\n\t" + "ADDS r8, r10, r11\n\t" + "ADDS r11, r1, #19\n\t" + "ADCS r11, r2, #0\n\t" + "ADCS r11, r3, #0\n\t" + "ADCS r11, r4, #0\n\t" + "ADCS r11, r5, #0\n\t" + "ADCS r11, r6, #0\n\t" + "ADCS r11, r7, #0\n\t" + "ADC r11, r8, #0\n\t" + "ASR r11, r11, #31\n\t" + "AND r11, r11, #19\n\t" + "ADDS r1, r1, r11\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r8, r8, #0\n\t" + "BFC r8, #31, #1\n\t" + /* Store */ + "LDR r0, [sp, #64]\n\t" + "STM r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" + "ADD sp, sp, #0x44\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op() +WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2738,6 +3612,155 @@ WC_OMIT_FRAME_POINTER void fe_sq_op() ); } +void fe_sq_op_full_red(void); +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#else +WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + __asm__ __volatile__ ( + "SUB sp, sp, #32\n\t" + "STR r0, [sp, #28]\n\t" + "LDM r1, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" + /* Square */ + "UMULL r9, r10, r0, r0\n\t" + "UMULL r11, r12, r0, r1\n\t" + "ADDS r11, r11, r11\n\t" + "MOV lr, #0\n\t" + "UMAAL r10, r11, lr, lr\n\t" + "STM sp, {r9, r10}\n\t" + "MOV r8, lr\n\t" + "UMAAL r8, r12, r0, r2\n\t" + "ADCS r8, r8, r8\n\t" + "UMAAL r8, r11, r1, r1\n\t" + "UMULL r9, r10, r0, r3\n\t" + "UMAAL r9, r12, r1, r2\n\t" + "ADCS r9, r9, r9\n\t" + "UMAAL r9, r11, lr, lr\n\t" + "STRD r8, r9, [sp, #8]\n\t" + "MOV r9, lr\n\t" + "UMAAL r9, r10, r0, r4\n\t" + "UMAAL r9, r12, r1, r3\n\t" + "ADCS r9, r9, r9\n\t" + "UMAAL r9, r11, r2, r2\n\t" + "STR r9, [sp, #16]\n\t" + "UMULL r9, r8, r0, r5\n\t" + "UMAAL r9, r12, r1, r4\n\t" + "UMAAL r9, r10, r2, r3\n\t" + "ADCS r9, r9, r9\n\t" + "UMAAL r9, r11, lr, lr\n\t" + "STR r9, [sp, #20]\n\t" + "MOV r9, lr\n\t" + "UMAAL r9, r8, r0, r6\n\t" + "UMAAL r9, r12, r1, r5\n\t" + "UMAAL r9, r10, r2, r4\n\t" + "ADCS r9, r9, r9\n\t" + "UMAAL r9, r11, r3, r3\n\t" + "STR r9, [sp, #24]\n\t" + "UMULL r0, r9, r0, r7\n\t" + "UMAAL r0, r8, r1, r6\n\t" + "UMAAL r0, r12, r2, r5\n\t" + "UMAAL r0, r10, r3, r4\n\t" + "ADCS r0, r0, r0\n\t" + "UMAAL r0, r11, lr, lr\n\t" + /* R[7] = r0 */ + "UMAAL r9, r8, r1, r7\n\t" + "UMAAL r9, r10, r2, r6\n\t" + "UMAAL r12, r9, r3, r5\n\t" + "ADCS r12, r12, r12\n\t" + "UMAAL r12, r11, r4, r4\n\t" + /* R[8] = r12 */ + "UMAAL r9, r8, r2, r7\n\t" + "UMAAL r10, r9, r3, r6\n\t" + "MOV r2, lr\n\t" + "UMAAL r10, r2, r4, r5\n\t" + "ADCS r10, r10, r10\n\t" + "UMAAL r11, r10, lr, lr\n\t" + /* R[9] = r11 */ + "UMAAL r2, r8, r3, r7\n\t" + "UMAAL r2, r9, r4, r6\n\t" + "ADCS r3, r2, r2\n\t" + "UMAAL r10, r3, r5, r5\n\t" + /* R[10] = r10 */ + "MOV r1, lr\n\t" + "UMAAL r1, r8, r4, r7\n\t" + "UMAAL r1, r9, r5, r6\n\t" + "ADCS r4, r1, r1\n\t" + "UMAAL r3, r4, lr, lr\n\t" + /* R[11] = r3 */ + "UMAAL r8, r9, r5, r7\n\t" + "ADCS r8, r8, r8\n\t" + "UMAAL r4, r8, r6, r6\n\t" + /* R[12] = r4 */ + "MOV r5, lr\n\t" + "UMAAL r5, r9, r6, r7\n\t" + "ADCS r5, r5, r5\n\t" + "UMAAL r8, r5, lr, lr\n\t" + /* R[13] = r8 */ + "ADCS r9, r9, r9\n\t" + "UMAAL r9, r5, r7, r7\n\t" + "ADCS r7, r5, lr\n\t" + /* R[14] = r9 */ + /* R[15] = r7 */ + /* Reduce */ + "MOV r6, #37\n\t" + "UMAAL r7, r0, r7, r6\n\t" + "MOV r6, #19\n\t" + "LSL r0, r0, #1\n\t" + "ORR r0, r0, r7, LSR #31\n\t" + "MUL lr, r0, r6\n\t" + "POP {r0, r1}\n\t" + "MOV r6, #38\n\t" + "UMAAL r0, lr, r12, r6\n\t" + "UMAAL r1, lr, r11, r6\n\t" + "MOV r12, r3\n\t" + "MOV r11, r4\n\t" + "POP {r2, r3, r4}\n\t" + "UMAAL r2, lr, r10, r6\n\t" + "UMAAL r3, lr, r12, r6\n\t" + "UMAAL r4, lr, r11, r6\n\t" + "MOV r12, r6\n\t" + "POP {r5, r6}\n\t" + "UMAAL r5, lr, r8, r12\n\t" + "BFC r7, #31, #1\n\t" + "UMAAL r6, lr, r9, r12\n\t" + "ADD r7, r7, lr\n\t" + "ADDS lr, r0, #19\n\t" + "ADCS lr, r1, #0\n\t" + "ADCS lr, r2, #0\n\t" + "ADCS lr, r3, #0\n\t" + "ADCS lr, r4, #0\n\t" + "ADCS lr, r5, #0\n\t" + "ADCS lr, r6, #0\n\t" + "ADC lr, r7, #0\n\t" + "ASR lr, lr, #31\n\t" + "AND lr, lr, #19\n\t" + "ADDS r0, r0, lr\n\t" + "ADCS r1, r1, #0\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADC r7, r7, #0\n\t" + "BFC r7, #31, #1\n\t" + "POP {lr}\n\t" + /* Store */ + "STM lr, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : + : +#else + : + : +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "lr" + ); +} + #endif /* WOLFSSL_ARM_ARCH_7M */ #ifndef WOLFSSL_NO_VAR_ASSIGN_REG WC_OMIT_FRAME_POINTER void fe_sq(fe r_p, const fe a_p) @@ -3068,17 +4091,17 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "LDR r2, [sp, #160]\n\t" "ADD r1, sp, #0x60\n\t" "ADD r0, sp, #32\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x80\n\t" "MOV r1, sp\n\t" "MOV r0, sp\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r1, sp, #0x80\n\t" "ADD r0, sp, #0x80\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "LDR r1, [sp, #160]\n\t" "ADD r0, sp, #0x60\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "MOV r3, sp\n\t" "ADD r2, sp, #32\n\t" "MOV r1, sp\n\t" @@ -3087,7 +4110,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "ADD r2, sp, #0x80\n\t" "ADD r1, sp, #0x60\n\t" "LDR r0, [sp, #160]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x80\n\t" "ADD r1, sp, #0x60\n\t" "ADD r0, sp, #0x60\n\t" @@ -3100,7 +4123,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "BL fe_mul121666\n\t" "ADD r1, sp, #0x40\n\t" "ADD r0, sp, #0x40\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "ADD r2, sp, #32\n\t" "ADD r1, sp, #0x80\n\t" "ADD r0, sp, #0x80\n\t" @@ -3108,11 +4131,11 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "MOV r2, sp\n\t" "LDR r1, [sp, #168]\n\t" "ADD r0, sp, #32\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x80\n\t" "ADD r1, sp, #0x60\n\t" "MOV r0, sp\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "LDR %[a], [sp, #176]\n\t" "LDR %[n], [sp, #180]\n\t" "SUBS %[n], %[n], #1\n\t" @@ -3497,17 +4520,17 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "LDR r2, [sp, #176]\n\t" "ADD r1, sp, #0x60\n\t" "LDR r0, [sp, #188]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x80\n\t" "LDR r1, [sp, #184]\n\t" "LDR r0, [sp, #184]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r1, sp, #0x80\n\t" "ADD r0, sp, #0x60\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "LDR r1, [sp, #176]\n\t" "ADD r0, sp, #0x80\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "LDR r3, [sp, #184]\n\t" "LDR r2, [sp, #188]\n\t" "LDR r1, [sp, #184]\n\t" @@ -3516,7 +4539,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "ADD r2, sp, #0x60\n\t" "ADD r1, sp, #0x80\n\t" "LDR r0, [sp, #176]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x60\n\t" "ADD r1, sp, #0x80\n\t" "ADD r0, sp, #0x80\n\t" @@ -3529,7 +4552,7 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "BL fe_mul121666\n\t" "LDR r1, [sp, #180]\n\t" "LDR r0, [sp, #180]\n\t" - "BL fe_sq_op\n\t" + "BL fe_sq_op_full_red\n\t" "LDR r2, [sp, #188]\n\t" "ADD r1, sp, #0x60\n\t" "ADD r0, sp, #0x60\n\t" @@ -3537,11 +4560,11 @@ WC_OMIT_FRAME_POINTER int curve25519(byte* r, const byte* n, const byte* a) "LDR r2, [sp, #184]\n\t" "LDR r1, [sp, #172]\n\t" "LDR r0, [sp, #188]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "ADD r2, sp, #0x60\n\t" "ADD r1, sp, #0x80\n\t" "LDR r0, [sp, #184]\n\t" - "BL fe_mul_op\n\t" + "BL fe_mul_op_full_red\n\t" "LDR %[a], [sp, #168]\n\t" "SUBS %[a], %[a], #1\n\t" #if defined(__GNUC__) @@ -4399,6 +5422,25 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) "BFC r8, #31, #1\n\t" "ADCS r7, r7, #0\n\t" "ADC r8, r8, #0\n\t" + "ADDS r11, r1, #19\n\t" + "ADCS r11, r2, #0\n\t" + "ADCS r11, r3, #0\n\t" + "ADCS r11, r4, #0\n\t" + "ADCS r11, r5, #0\n\t" + "ADCS r11, r6, #0\n\t" + "ADCS r11, r7, #0\n\t" + "ADC r11, r8, #0\n\t" + "ASR r11, r11, #31\n\t" + "AND r11, r11, #19\n\t" + "ADDS r1, r1, r11\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADCS r7, r7, #0\n\t" + "ADC r8, r8, #0\n\t" + "BFC r8, #31, #1\n\t" /* Store */ "LDR r0, [sp, #64]\n\t" "STM r0, {r1, r2, r3, r4, r5, r6, r7, r8}\n\t" @@ -4566,6 +5608,25 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) "BFC r7, #31, #1\n\t" "ADCS r6, r6, #0\n\t" "ADC r7, r7, #0\n\t" + "ADDS r12, r0, #19\n\t" + "ADCS r12, r1, #0\n\t" + "ADCS r12, r2, #0\n\t" + "ADCS r12, r3, #0\n\t" + "ADCS r12, r4, #0\n\t" + "ADCS r12, r5, #0\n\t" + "ADCS r12, r6, #0\n\t" + "ADC r12, r7, #0\n\t" + "ASR r12, r12, #31\n\t" + "AND r12, r12, #19\n\t" + "ADDS r0, r0, r12\n\t" + "ADCS r1, r1, #0\n\t" + "ADCS r2, r2, #0\n\t" + "ADCS r3, r3, #0\n\t" + "ADCS r4, r4, #0\n\t" + "ADCS r5, r5, #0\n\t" + "ADCS r6, r6, #0\n\t" + "ADC r7, r7, #0\n\t" + "BFC r7, #31, #1\n\t" "POP {r12, lr}\n\t" /* Store */ "STM r12, {r0, r1, r2, r3, r4, r5, r6, r7}\n\t" diff --git a/wolfcrypt/src/sp_arm32.c b/wolfcrypt/src/sp_arm32.c index beb0134c91..b28a365439 100644 --- a/wolfcrypt/src/sp_arm32.c +++ b/wolfcrypt/src/sp_arm32.c @@ -11951,9 +11951,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_2048_word_32(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -16382,9 +16381,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_2048_word_64(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -37688,9 +37686,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_3072_word_48(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -43807,9 +43804,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_3072_word_96(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -59105,9 +59101,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_4096_word_128(sp_digit d1, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -79263,9 +79258,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_256_word_8(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -97707,9 +97701,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_384_word_12(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -126361,9 +126354,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_521_word_17(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -146927,9 +146919,8 @@ WC_OMIT_FRAME_POINTER static sp_digit div_1024_word_32(sp_digit d1, sp_digit d0, "subs r7, %[d0], r4\n\t" "sbc r8, %[d1], r5\n\t" "add r3, r3, r8\n\t" - "subs r6, %[div], r7\n\t" - "sbc r6, r6, r6\n\t" - "sub %[d1], r3, r6\n\t" + "subs r6, r7, %[div]\n\t" + "adc %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : diff --git a/wolfcrypt/src/sp_armthumb.c b/wolfcrypt/src/sp_armthumb.c index 078be16087..b7dd33133e 100644 --- a/wolfcrypt/src/sp_armthumb.c +++ b/wolfcrypt/src/sp_armthumb.c @@ -23915,11 +23915,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -23929,9 +23929,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -27389,11 +27394,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -27403,9 +27408,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -75363,11 +75373,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -75377,9 +75387,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -79663,11 +79678,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -79677,9 +79692,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -92180,11 +92200,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -92194,9 +92214,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -105145,11 +105170,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -105159,9 +105184,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -115944,11 +115974,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -115958,9 +115988,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -131764,11 +131799,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -131778,9 +131813,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) @@ -201799,11 +201839,11 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1, #else "add r3, r3, r5\n\t" #endif - "movs r6, %[div]\n\t" + "movs r6, r4\n\t" #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r6, r6, r4\n\t" + "subs r6, r6, %[div]\n\t" #else - "sub r6, r6, r4\n\t" + "sub r6, r6, %[div]\n\t" #endif #ifdef WOLFSSL_KEIL "sbcs r6, r6, r6\n\t" @@ -201813,9 +201853,14 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1, "sbc r6, r6\n\t" #endif #if defined(__clang__) || defined(WOLFSSL_KEIL) - "subs r3, r3, r6\n\t" + "adds r3, r3, #1\n\t" #else - "sub r3, r3, r6\n\t" + "add r3, r3, #1\n\t" +#endif +#if defined(__clang__) || defined(WOLFSSL_KEIL) + "adds r3, r3, r6\n\t" +#else + "add r3, r3, r6\n\t" #endif "movs %[d1], r3\n\t" : [d1] "+l" (d1), [d0] "+l" (d0), [div] "+l" (div) diff --git a/wolfcrypt/src/sp_c32.c b/wolfcrypt/src/sp_c32.c index eb87e8ed50..00b2225961 100644 --- a/wolfcrypt/src/sp_c32.c +++ b/wolfcrypt/src/sp_c32.c @@ -26258,6 +26258,8 @@ static int sp_256_div_9(const sp_digit* a, const sp_digit* d, sp_256_norm_9(&t1[i + 1]); } sp_256_norm_9(t1); + sp_256_cond_add_9(t1, t1, sd, t1[8] >> 31); + sp_256_norm_9(t1); sp_256_rshift_9(r, t1, 5); } @@ -34192,6 +34194,8 @@ static int sp_384_div_15(const sp_digit* a, const sp_digit* d, sp_384_norm_15(&t1[i + 1]); } sp_384_norm_15(t1); + sp_384_cond_add_15(t1, t1, sd, t1[14] >> 31); + sp_384_norm_15(t1); sp_384_rshift_15(r, t1, 6); } @@ -42165,6 +42169,8 @@ static int sp_521_div_21(const sp_digit* a, const sp_digit* d, sp_521_norm_21(&t1[i + 1]); } sp_521_norm_21(t1); + sp_521_cond_add_21(t1, t1, sd, t1[20] >> 31); + sp_521_norm_21(t1); sp_521_rshift_21(r, t1, 4); } diff --git a/wolfcrypt/src/sp_c64.c b/wolfcrypt/src/sp_c64.c index 06ddef7acc..539ade5eb5 100644 --- a/wolfcrypt/src/sp_c64.c +++ b/wolfcrypt/src/sp_c64.c @@ -34293,6 +34293,8 @@ static int sp_384_div_7(const sp_digit* a, const sp_digit* d, sp_384_norm_7(&t1[i + 1]); } sp_384_norm_7(t1); + sp_384_cond_add_7(t1, t1, sd, t1[6] >> 63); + sp_384_norm_7(t1); sp_384_rshift_7(r, t1, 1); } @@ -41580,6 +41582,8 @@ static int sp_521_div_9(const sp_digit* a, const sp_digit* d, sp_521_norm_9(&t1[i + 1]); } sp_521_norm_9(t1); + sp_521_cond_add_9(t1, t1, sd, t1[8] >> 63); + sp_521_norm_9(t1); sp_521_rshift_9(r, t1, 1); } diff --git a/wolfcrypt/src/sp_cortexm.c b/wolfcrypt/src/sp_cortexm.c index e315322f7c..101c974bf1 100644 --- a/wolfcrypt/src/sp_cortexm.c +++ b/wolfcrypt/src/sp_cortexm.c @@ -5047,9 +5047,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_32(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -7786,9 +7785,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_2048_word_64(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -16836,9 +16834,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_48(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -20335,9 +20332,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_3072_word_96(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -28485,9 +28481,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_4096_word_128(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -41062,9 +41057,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_256_word_8(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -51466,9 +51460,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_384_word_12(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -64807,9 +64800,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_521_word_17(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : @@ -71609,9 +71601,8 @@ WC_OMIT_FRAME_POINTER static SP_NOINLINE sp_digit div_1024_word_32(sp_digit d1, "SUBS r9, %[d0], r6\n\t" "SBC r10, %[d1], r7\n\t" "ADD r3, r3, r10\n\t" - "SUBS r8, %[div], r9\n\t" - "SBC r8, r8, r8\n\t" - "SUB %[d1], r3, r8\n\t" + "SUBS r8, r9, %[div]\n\t" + "ADC %[d1], r3, #0\n\t" #ifndef WOLFSSL_NO_VAR_ASSIGN_REG : [d1] "+r" (d1), [d0] "+r" (d0), [div] "+r" (div) : diff --git a/wolfcrypt/src/sp_int.c b/wolfcrypt/src/sp_int.c index 09856ef793..4400d6f845 100644 --- a/wolfcrypt/src/sp_int.c +++ b/wolfcrypt/src/sp_int.c @@ -5929,6 +5929,35 @@ int sp_cmp_ct(const sp_int* a, const sp_int* b, unsigned int n) } #endif /* HAVE_ECC && !WC_NO_RNG && WOLFSSL_ECC_GEN_REJECT_SAMPLING */ +/* Constant time clamping. + * + * @param [in, out] a SP integer to clamp. + */ +static void sp_clamp_ct(sp_int* a) +{ + int i; + sp_size_t used = a->used; + volatile sp_size_t mask = (sp_size_t)-1; + + for (i = (int)a->used - 1; i >= 0; i--) { +#if ((SP_WORD_SIZE == 64) && \ + (defined(_WIN64) || !defined(WOLFSSL_UINT128_T_DEFINED))) || \ + ((SP_WORD_SIZE == 32) && defined(NO_64BIT)) + sp_int_digit negVal = ~a->dp[i]; + sp_int_digit minusOne = a->dp[i] - 1; + sp_int_digit zeroMask = + (sp_int_digit)((sp_int_sdigit)(negVal & minusOne) >> + (SP_WORD_SIZE - 1)); +#else + sp_size_t zeroMask = + (sp_size_t)((((sp_int_sword)a->dp[i]) - 1) >> SP_WORD_SIZE); +#endif + mask &= (sp_size_t)zeroMask; + used = (sp_size_t)(used + mask); + } + a->used = used; +} + /************************* * Bit check/set functions *************************/ @@ -7550,6 +7579,9 @@ int sp_div_2_mod_ct(const sp_int* a, const sp_int* m, sp_int* r) /* Divide conditional sum by 2. */ _sp_div_2(r, r); + /* Remove leading zeros. */ + sp_clamp_ct(r); + #if 0 sp_print(r, "rd2"); #endif @@ -8133,35 +8165,6 @@ int sp_submod(const sp_int* a, const sp_int* b, const sp_int* m, sp_int* r) } #endif /* WOLFSSL_SP_MATH_ALL */ -/* Constant time clamping. - * - * @param [in, out] a SP integer to clamp. - */ -static void sp_clamp_ct(sp_int* a) -{ - int i; - sp_size_t used = a->used; - volatile sp_size_t mask = (sp_size_t)-1; - - for (i = (int)a->used - 1; i >= 0; i--) { -#if ((SP_WORD_SIZE == 64) && \ - (defined(_WIN64) || !defined(WOLFSSL_UINT128_T_DEFINED))) || \ - ((SP_WORD_SIZE == 32) && defined(NO_64BIT)) - sp_int_digit negVal = ~a->dp[i]; - sp_int_digit minusOne = a->dp[i] - 1; - sp_int_digit zeroMask = - (sp_int_digit)((sp_int_sdigit)(negVal & minusOne) >> - (SP_WORD_SIZE - 1)); -#else - sp_size_t zeroMask = - (sp_size_t)((((sp_int_sword)a->dp[i]) - 1) >> SP_WORD_SIZE); -#endif - mask &= (sp_size_t)zeroMask; - used = (sp_size_t)(used + mask); - } - a->used = used; -} - #if defined(WOLFSSL_SP_MATH_ALL) && defined(HAVE_ECC) /* Add two values and reduce: r = (a + b) % m * diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 56d1af0a04..8431fb3ae6 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -5441,8 +5441,9 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i, r = w1[j] + ((0 - (((word32)w1[j]) >> 31)) & MLDSA_Q); /* Decompose value into low and high parts. */ mldsa_decompose_q88(r, &r0, &r1); - /* Make hint positive or negative based on sign of r0. */ - hint = (1 - (2 * (((word32)r0) >> 31))) & (0 - hint); + /* Make hint positive or negative: r0 > 0 adds, r0 <= 0 subtracts + * (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */ + hint = (1 - (2 * (((word32)(r0 - 1)) >> 31))) & (0 - hint); /* Make w1 only the top part plus the hint. */ w1[j] = r1 + hint; @@ -5457,8 +5458,9 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i, mldsa_decompose_q88(r, &r0, &r1); /* Check for hint. */ if ((o < h[PARAMS_ML_DSA_44_OMEGA + i]) && (h[o] == (byte)j)) { - /* Add or subtract hint based on sign of r0. */ - r1 += (sword32)(1U - (2U * (((word32)r0) >> 31))); + /* Add or subtract hint: r0 > 0 adds, r0 <= 0 subtracts + * (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */ + r1 += (sword32)(1U - (2U * (((word32)(r0 - 1)) >> 31))); /* Go to next hint offset. */ o++; } @@ -5511,8 +5513,9 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega, r = w1[j] + (sword32)((0 - (((word32)w1[j]) >> 31)) & MLDSA_Q); /* Decompose value into low and high parts. */ mldsa_decompose_q32(r, &r0, &r1); - /* Make hint positive or negative based on sign of r0. */ - hint = (sword32)((1 - (2 * (((word32)r0) >> 31))) & (0 - hint)); + /* Make hint positive or negative: r0 > 0 adds, r0 <= 0 subtracts + * (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */ + hint = (sword32)((1 - (2 * (((word32)(r0 - 1)) >> 31))) & (0 - hint)); /* Make w1 only the top part plus the hint. */ w1[j] = r1 + hint; @@ -5525,8 +5528,9 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega, mldsa_decompose_q32(r, &r0, &r1); /* Check for hint. */ if ((o < h[omega + i]) && (h[o] == (byte)j)) { - /* Add or subtract hint based on sign of r0. */ - r1 += (sword32)(1 - (2 * (((word32)r0) >> 31))); + /* Add or subtract hint: r0 > 0 adds, r0 <= 0 subtracts + * (FIPS 204). (r0 - 1) >> 31 is 1 when r0 <= 0. */ + r1 += (sword32)(1 - (2 * (((word32)(r0 - 1)) >> 31))); /* Go to next hint offset. */ o++; } diff --git a/wolfcrypt/src/wc_mldsa_asm.S b/wolfcrypt/src/wc_mldsa_asm.S index e1e77a9378..1c6e5190ee 100644 --- a/wolfcrypt/src/wc_mldsa_asm.S +++ b/wolfcrypt/src/wc_mldsa_asm.S @@ -27987,6 +27987,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_0: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28071,6 +28073,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_1: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28155,6 +28159,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_2: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28239,6 +28245,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_3: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28323,6 +28331,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_4: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28407,6 +28417,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_5: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28491,6 +28503,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_6: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28575,6 +28589,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_7: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28659,6 +28675,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_8: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28743,6 +28761,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_9: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28827,6 +28847,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_10: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28911,6 +28933,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_11: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -28995,6 +29019,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_12: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29079,6 +29105,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_13: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29163,6 +29191,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_14: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29247,6 +29277,8 @@ L_mldsa_use_hint_88_avx2_hints_done_0_15: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29332,6 +29364,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_0: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29416,6 +29450,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_1: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29500,6 +29536,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_2: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29584,6 +29622,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_3: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29668,6 +29708,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_4: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29752,6 +29794,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_5: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29836,6 +29880,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_6: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -29920,6 +29966,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_7: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30004,6 +30052,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_8: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30088,6 +30138,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_9: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30172,6 +30224,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_10: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30256,6 +30310,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_11: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30340,6 +30396,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_12: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30424,6 +30482,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_13: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30508,6 +30568,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_14: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30592,6 +30654,8 @@ L_mldsa_use_hint_88_avx2_hints_done_1_15: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30677,6 +30741,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_0: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30761,6 +30827,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_1: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30845,6 +30913,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_2: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -30929,6 +30999,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_3: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31013,6 +31085,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_4: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31097,6 +31171,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_5: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31181,6 +31257,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_6: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31265,6 +31343,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_7: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31349,6 +31429,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_8: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31433,6 +31515,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_9: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31517,6 +31601,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_10: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31601,6 +31687,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_11: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31685,6 +31773,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_12: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31769,6 +31859,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_13: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31853,6 +31945,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_14: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -31937,6 +32031,8 @@ L_mldsa_use_hint_88_avx2_hints_done_2_15: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32022,6 +32118,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_0: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32106,6 +32204,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_1: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32190,6 +32290,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_2: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32274,6 +32376,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_3: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32358,6 +32462,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_4: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32442,6 +32548,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_5: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32526,6 +32634,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_6: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32610,6 +32720,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_7: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32694,6 +32806,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_8: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32778,6 +32892,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_9: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32862,6 +32978,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_10: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -32946,6 +33064,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_11: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33030,6 +33150,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_12: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33114,6 +33236,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_13: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33198,6 +33322,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_14: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33282,6 +33408,8 @@ L_mldsa_use_hint_88_avx2_hints_done_3_15: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33486,6 +33614,8 @@ L_mldsa_use_hint_32_avx2_hints_done__0: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33558,6 +33688,8 @@ L_mldsa_use_hint_32_avx2_hints_done__1: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33630,6 +33762,8 @@ L_mldsa_use_hint_32_avx2_hints_done__2: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33702,6 +33836,8 @@ L_mldsa_use_hint_32_avx2_hints_done__3: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33774,6 +33910,8 @@ L_mldsa_use_hint_32_avx2_hints_done__4: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33846,6 +33984,8 @@ L_mldsa_use_hint_32_avx2_hints_done__5: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33918,6 +34058,8 @@ L_mldsa_use_hint_32_avx2_hints_done__6: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -33990,6 +34132,8 @@ L_mldsa_use_hint_32_avx2_hints_done__7: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34062,6 +34206,8 @@ L_mldsa_use_hint_32_avx2_hints_done__8: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34134,6 +34280,8 @@ L_mldsa_use_hint_32_avx2_hints_done__9: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34206,6 +34354,8 @@ L_mldsa_use_hint_32_avx2_hints_done__10: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34278,6 +34428,8 @@ L_mldsa_use_hint_32_avx2_hints_done__11: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34350,6 +34502,8 @@ L_mldsa_use_hint_32_avx2_hints_done__12: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34422,6 +34576,8 @@ L_mldsa_use_hint_32_avx2_hints_done__13: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34494,6 +34650,8 @@ L_mldsa_use_hint_32_avx2_hints_done__14: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2 @@ -34566,6 +34724,8 @@ L_mldsa_use_hint_32_avx2_hints_done__15: vpsllvd %ymm13, %ymm7, %ymm7 vpsrad $31, %ymm6, %ymm6 vpsrad $31, %ymm7, %ymm7 + vpsubd %ymm14, %ymm2, %ymm2 + vpsubd %ymm14, %ymm3, %ymm3 vpsrld $31, %ymm2, %ymm2 vpsrld $31, %ymm3, %ymm3 vpslld $0x01, %ymm2, %ymm2